From 82122530e846b208a8434467a12345076f41e195 Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Wed, 9 Jul 2025 16:56:31 +0100 Subject: [PATCH 1/9] wip --- Cargo.lock | 33 ++- Cargo.toml | 3 +- helix-cli/src/args.rs | 2 +- helix-cli/src/main.rs | 2 + helix-container/Cargo.toml | 12 +- helix-container/src/main.rs | 4 +- helixdb/.cargo/config.toml | 2 + helixdb/Cargo.toml | 1 + helixdb/src/helix_gateway/router/dynamic.rs | 62 +++++ helixdb/src/helix_gateway/router/mod.rs | 17 ++ helixdb/src/helix_gateway/router/router.rs | 14 +- query-container/.cargo/config.toml | 2 + query-container/Cargo.toml | 18 ++ query-container/src/lib.rs | 41 ++++ query-container/src/query.rs | 245 ++++++++++++++++++++ 15 files changed, 434 insertions(+), 24 deletions(-) create mode 100644 helixdb/.cargo/config.toml create mode 100644 helixdb/src/helix_gateway/router/dynamic.rs create mode 100644 query-container/.cargo/config.toml create mode 100644 query-container/Cargo.toml create mode 100644 query-container/src/lib.rs create mode 100644 query-container/src/query.rs diff --git a/Cargo.lock b/Cargo.lock index 1c6b27dd..a70a2e5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytecheck" @@ -1983,6 +1983,7 @@ dependencies = [ "itertools 0.14.0", "kdam", "lazy_static", + "libloading", "native-tls", "pest", "pest_derive", @@ -2536,12 +2537,12 @@ checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] @@ -3782,6 +3783,24 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "query-container" +version = "0.1.0" +dependencies = [ + "chrono", + "dirs 5.0.1", + "get_routes", + "heed3", + "helixdb", + "inventory", + "rand 0.9.1", + "serde", + "serde_json", + "sonic-rs", + "tokio", + "uuid", +] + [[package]] name = "quote" version = "1.0.40" @@ -4622,9 +4641,9 @@ dependencies = [ [[package]] name = "sonic-rs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93070f7e7c0d7ec7d08406b1b407234af30420320fd854f304029e3c6db4a899" +checksum = "7be54789747a46a8b1eb7b2c9cb0879cd7559a5f71bfff950868369f6868b9ad" dependencies = [ "ahash 0.8.11", "bumpalo", diff --git a/Cargo.toml b/Cargo.toml index cad630dc..41013391 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "helix-cli", "hbuild", "hbuild_redploy", + "query-container", ] resolver = "2" @@ -23,4 +24,4 @@ opt-level = 0 codegen-units = 256 incremental = true panic = "abort" -debug = 1 +debug = 1 diff --git a/helix-cli/src/args.rs b/helix-cli/src/args.rs index e34c2681..2ca64e8a 100644 --- a/helix-cli/src/args.rs +++ b/helix-cli/src/args.rs @@ -79,7 +79,7 @@ pub struct DeployCommand { #[clap(short, long, help = "The output path")] pub output: Option, - #[clap(short, long, help = "Port to run the instance on")] + #[clap(long, help = "Port to run the instance on")] pub port: Option, } diff --git a/helix-cli/src/main.rs b/helix-cli/src/main.rs index e69a11c3..817d509c 100644 --- a/helix-cli/src/main.rs +++ b/helix-cli/src/main.rs @@ -288,6 +288,8 @@ fi } } + println!("\n\n{generated_rust_code}\n\n"); + match fs::write(file_path, generated_rust_code) { Ok(_) => { println!("{}", "Successfully wrote queries file".green().bold()); diff --git a/helix-container/Cargo.toml b/helix-container/Cargo.toml index 1b2a20cb..4bed39a0 100644 --- a/helix-container/Cargo.toml +++ b/helix-container/Cargo.toml @@ -17,12 +17,12 @@ serde_json = "1.0.140" uuid = { version = "1.12.1", features = ["std", "v4", "v6", "fast-rng"] } heed3 = "0.22.0" -[profile.release] -strip = "debuginfo" -lto = true -opt-level = 3 -codegen-units = 1 -panic = "abort" +# [profile.release] +# strip = "debuginfo" +# lto = true +# opt-level = 3 +# codegen-units = 1 +# panic = "abort" [features] dev = ["helixdb/dev"] diff --git a/helix-container/src/main.rs b/helix-container/src/main.rs index 6a4d56af..7a17c772 100644 --- a/helix-container/src/main.rs +++ b/helix-container/src/main.rs @@ -60,8 +60,7 @@ async fn main() { .map(|submission| { println!("Processing submission for handler: {}", submission.0.name); let handler = &submission.0; - let func: HandlerFn = - Arc::new(move |input, response| (handler.func)(input, response)); + let func: HandlerFn = Arc::new(handler.func); ( ( "post".to_ascii_uppercase().to_string(), @@ -111,4 +110,3 @@ async fn main() { let a = gateway.connection_handler.accept_conns().await.unwrap(); let _b = a.await.unwrap(); } - diff --git a/helixdb/.cargo/config.toml b/helixdb/.cargo/config.toml new file mode 100644 index 00000000..52e8b726 --- /dev/null +++ b/helixdb/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["-C", "link-args=-rdynamic"] diff --git a/helixdb/Cargo.toml b/helixdb/Cargo.toml index 5a5253a4..d15a31f3 100644 --- a/helixdb/Cargo.toml +++ b/helixdb/Cargo.toml @@ -45,6 +45,7 @@ tokio-postgres = { version = "0.7", features = [ "with-uuid-1", "with-chrono-0_4", ], optional = true } +libloading = "0.8.8" [dev-dependencies] diff --git a/helixdb/src/helix_gateway/router/dynamic.rs b/helixdb/src/helix_gateway/router/dynamic.rs new file mode 100644 index 00000000..d1ae256d --- /dev/null +++ b/helixdb/src/helix_gateway/router/dynamic.rs @@ -0,0 +1,62 @@ +use libloading::{self, Library, Symbol}; +use std::{error::Error, ops::Deref, path::PathBuf, sync::Arc}; + +use crate::{ + helix_engine::types::GraphError, + helix_gateway::router::{ + router::{HandlerInput, HelixRouter}, + QueryHandler, + }, + protocol::response::Response, +}; + +#[derive(Clone)] +pub struct DynHandler { + // holding this guarentees that the Symbol is still valid + _source: Arc, + func: extern "Rust" fn(&HandlerInput, &mut Response) -> Result<(), GraphError>, +} + +impl QueryHandler for DynHandler { + fn handle(&self, input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { + (self.func)(input, response) + } +} + +// impl QueryHandler for DynHandler { +// fn handle(&self, input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { +// self.func. +// } +// } + +type DynQueryFn = extern "Rust" fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; +type GetQueryFn = extern "Rust" fn() -> Vec<(String, DynQueryFn)>; + +struct Plugin { + lib: Arc, +} + +impl Plugin { + /// SAFETY: This must be called with a path to Helix query dynamic library, compiled with the same version of Rust as the main database + unsafe fn open(lib_path: PathBuf) -> Result> { + let lib = Library::new(lib_path)?; + Ok(Plugin { lib: Arc::new(lib) }) + } + + fn add_queries(&self, router: &mut HelixRouter) -> Result<(), Box> { + // SAFETY: If a valid file was opened it will have a get_queries function of this type + let get_fn: Symbol = unsafe { self.lib.get(b"get_queries")? }; + + let queries = get_fn(); + + for (name, func) in queries { + let handler = DynHandler { + _source: self.lib.clone(), + func, + }; + router.add_route("post", &format!("/{name}"), Arc::new(handler)); + } + + todo!() + } +} diff --git a/helixdb/src/helix_gateway/router/mod.rs b/helixdb/src/helix_gateway/router/mod.rs index fcddb5c6..1d2f940e 100644 --- a/helixdb/src/helix_gateway/router/mod.rs +++ b/helixdb/src/helix_gateway/router/mod.rs @@ -1 +1,18 @@ +use crate::{ + helix_engine::types::GraphError, + helix_gateway::router::router::{BasicHandlerFn, HandlerInput}, + protocol::response::Response, +}; + +mod dynamic; pub mod router; + +pub trait QueryHandler: Send + Sync { + fn handle(&self, input: &HandlerInput, response: &mut Response) -> Result<(), GraphError>; +} + +impl QueryHandler for BasicHandlerFn { + fn handle(&self, input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { + self(input, response) + } +} diff --git a/helixdb/src/helix_gateway/router/router.rs b/helixdb/src/helix_gateway/router/router.rs index a6665ab5..c401f8e3 100644 --- a/helixdb/src/helix_gateway/router/router.rs +++ b/helixdb/src/helix_gateway/router/router.rs @@ -9,7 +9,10 @@ use crate::{ helix_engine::{graph_core::graph_core::HelixGraphEngine, types::GraphError}, - helix_gateway::mcp::mcp::{MCPHandlerFn, MCPToolInput}, + helix_gateway::{ + mcp::mcp::{MCPHandlerFn, MCPToolInput}, + router::QueryHandler, + }, }; use core::fmt; use std::{collections::HashMap, sync::Arc}; @@ -25,8 +28,7 @@ pub struct HandlerInput { pub type BasicHandlerFn = fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; // thread safe type for multi threaded use -pub type HandlerFn = - Arc Result<(), GraphError> + Send + Sync>; +pub type HandlerFn = Arc; #[derive(Clone, Debug)] pub struct HandlerSubmission(pub Handler); @@ -75,9 +77,9 @@ impl HelixRouter { } /// Add a route to the router - pub fn add_route(&mut self, method: &str, path: &str, handler: BasicHandlerFn) { + pub fn add_route(&mut self, method: &str, path: &str, handler: HandlerFn) { self.routes - .insert((method.to_uppercase(), path.to_string()), Arc::new(handler)); + .insert((method.to_uppercase(), path.to_string()), handler); } /// Handle a request by finding the appropriate handler and executing it @@ -105,7 +107,7 @@ impl HelixRouter { request, graph: Arc::clone(&graph_access), }; - return handler(&input, response); + return handler.handle(&input, response); } if let Some(mcp_handler) = self.mcp_routes.get(&route_key) { diff --git a/query-container/.cargo/config.toml b/query-container/.cargo/config.toml new file mode 100644 index 00000000..f06ff793 --- /dev/null +++ b/query-container/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"] diff --git a/query-container/Cargo.toml b/query-container/Cargo.toml new file mode 100644 index 00000000..d543654b --- /dev/null +++ b/query-container/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "query-container" +version = "0.1.0" +edition = "2024" + +[dependencies] +helixdb = { path = "../helixdb" } +get_routes = { path = "../get_routes" } +inventory = "0.3.16" +rand = "0.9.1" +dirs = "5.0.1" +chrono = { version = "0.4.41", features = ["serde"] } +serde = { version = "1.0", features = ["derive"] } +sonic-rs = "0.5.0" +tokio = { version = "1.44.2", features = ["full"] } +serde_json = "1.0.140" +uuid = { version = "1.12.1", features = ["std", "v4", "v6", "fast-rng"] } +heed3 = "0.22.0" diff --git a/query-container/src/lib.rs b/query-container/src/lib.rs new file mode 100644 index 00000000..62c4cf43 --- /dev/null +++ b/query-container/src/lib.rs @@ -0,0 +1,41 @@ +use std::sync::Arc; + +use helixdb::{ + helix_engine::types::GraphError, + helix_gateway::router::{QueryHandler, router::HandlerInput}, + protocol::response::Response, +}; + +mod query; + +type DynQueryFn = fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; + +// basic type for function pointer +pub type BasicHandlerFn = fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; + +// thread safe type for multi threaded use +pub type HandlerFn = Arc; + +#[derive(Clone, Debug)] +pub struct HandlerSubmission(pub Handler); + +#[derive(Clone, Debug)] +pub struct Handler { + pub name: &'static str, + pub func: BasicHandlerFn, +} + +impl Handler { + pub const fn new(name: &'static str, func: BasicHandlerFn) -> Self { + Self { name, func } + } +} + +inventory::collect!(HandlerSubmission); + +#[unsafe(no_mangle)] +pub extern "Rust" fn get_queries() -> Vec<(String, DynQueryFn)> { + let submissions: Vec<_> = inventory::iter::.into_iter().collect(); + + todo!() +} diff --git a/query-container/src/query.rs b/query-container/src/query.rs new file mode 100644 index 00000000..883c605b --- /dev/null +++ b/query-container/src/query.rs @@ -0,0 +1,245 @@ +use chrono::{DateTime, Utc}; +use get_routes::handler; +use heed3::RoTxn; +use helixdb::helix_engine::vector_core::vector::HVector; +use helixdb::{ + exclude_field, field_remapping, identifier_remapping, traversal_remapping, value_remapping, +}; +use helixdb::{ + helix_engine::graph_core::ops::{ + bm25::search_bm25::SearchBM25Adapter, + g::G, + in_::{in_::InAdapter, in_e::InEdgesAdapter, to_n::ToNAdapter, to_v::ToVAdapter}, + out::{ + from_n::FromNAdapter, from_v::FromVAdapter, out::OutAdapter, out_e::OutEdgesAdapter, + }, + source::{ + add_e::{AddEAdapter, EdgeType}, + add_n::AddNAdapter, + e_from_id::EFromIdAdapter, + e_from_type::EFromTypeAdapter, + n_from_id::NFromIdAdapter, + n_from_index::NFromIndexAdapter, + n_from_type::NFromTypeAdapter, + }, + tr_val::{Traversable, TraversalVal}, + util::{ + dedup::DedupAdapter, drop::Drop, filter_mut::FilterMut, filter_ref::FilterRefAdapter, + map::MapAdapter, paths::ShortestPathAdapter, props::PropsAdapter, range::RangeAdapter, + update::UpdateAdapter, + }, + vectors::{ + brute_force_search::BruteForceSearchVAdapter, insert::InsertVAdapter, + search::SearchVAdapter, + }, + }, + helix_engine::types::GraphError, + helix_gateway::router::router::HandlerInput, + node_matches, props, + protocol::count::Count, + protocol::remapping::{RemappingMap, ResponseRemapping}, + protocol::response::Response, + protocol::{ + filterable::Filterable, id::ID, remapping::Remapping, return_values::ReturnValue, + value::Value, + }, +}; +use sonic_rs::{Deserialize, Serialize}; +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Instant; + +pub struct Cluster { + pub region: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub struct Instance { + pub region: String, + pub instance_type: String, + pub storage_gb: i64, + pub ram_gb: i64, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub struct User { + pub gh_id: u64, + pub gh_login: String, + pub name: String, + pub email: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub struct CreatedCluster { + pub from: User, + pub to: Cluster, +} + +pub struct CreatedInstance { + pub from: Cluster, + pub to: Instance, +} + +#[derive(Serialize, Deserialize)] +pub struct GetInstancesForUserInput { + pub user_id: ID, +} + +#[handler] +pub fn GetInstancesForUser( + input: &HandlerInput, + response: &mut Response, +) -> Result<(), GraphError> { + let data: GetInstancesForUserInput = match sonic_rs::from_slice(&input.request.body) { + Ok(data) => data, + Err(err) => return Err(GraphError::from(err)), + }; + + let mut remapping_vals = RemappingMap::new(); + let db = Arc::clone(&input.graph.storage); + let txn = db.graph_env.read_txn().unwrap(); + let instances = G::new(Arc::clone(&db), &txn) + .n_from_id(&data.user_id) + .out("CreatedCluster", &EdgeType::Node) + .out("CreatedInstance", &EdgeType::Node) + .collect_to::>(); + let mut return_vals: HashMap = HashMap::new(); + return_vals.insert( + "instances".to_string(), + ReturnValue::from_traversal_value_array_with_mixin( + instances.clone(), + remapping_vals.borrow_mut(), + ), + ); + + txn.commit().unwrap(); + response.body = sonic_rs::to_vec(&return_vals).unwrap(); + Ok(()) +} + +#[derive(Serialize, Deserialize)] +pub struct CreateUserInput { + pub gh_id: u64, + pub gh_login: String, + pub name: String, + pub email: String, +} +#[handler] +pub fn CreateUser(input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { + let data: CreateUserInput = match sonic_rs::from_slice(&input.request.body) { + Ok(data) => data, + Err(err) => return Err(GraphError::from(err)), + }; + + let mut remapping_vals = RemappingMap::new(); + let db = Arc::clone(&input.graph.storage); + let mut txn = db.graph_env.write_txn().unwrap(); + let user = G::new_mut(Arc::clone(&db), &mut txn) +.add_n("User", Some(props! { "gh_id" => data.gh_id.clone(), "gh_login" => data.gh_login.clone(), "email" => data.email.clone(), "created_at" => chrono::Utc::now().to_rfc3339(), "updated_at" => chrono::Utc::now().to_rfc3339(), "name" => data.name.clone() }), Some(&["gh_id"])).collect_to::>(); + let mut return_vals: HashMap = HashMap::new(); + return_vals.insert( + "user".to_string(), + ReturnValue::from_traversal_value_array_with_mixin( + user.clone(), + remapping_vals.borrow_mut(), + ), + ); + + txn.commit().unwrap(); + response.body = sonic_rs::to_vec(&return_vals).unwrap(); + Ok(()) +} + +#[derive(Serialize, Deserialize)] +pub struct LookupUserInput { + pub gh_id: u64, +} +#[handler] +pub fn LookupUser(input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { + let data: LookupUserInput = match sonic_rs::from_slice(&input.request.body) { + Ok(data) => data, + Err(err) => return Err(GraphError::from(err)), + }; + + let mut remapping_vals = RemappingMap::new(); + let db = Arc::clone(&input.graph.storage); + let txn = db.graph_env.read_txn().unwrap(); + let user = G::new(Arc::clone(&db), &txn) + .n_from_index("gh_id", &data.gh_id) + .collect_to::>(); + let mut return_vals: HashMap = HashMap::new(); + return_vals.insert( + "user".to_string(), + ReturnValue::from_traversal_value_array_with_mixin( + user.clone(), + remapping_vals.borrow_mut(), + ), + ); + + txn.commit().unwrap(); + response.body = sonic_rs::to_vec(&return_vals).unwrap(); + Ok(()) +} + +#[derive(Serialize, Deserialize)] +pub struct CreateClusterInput { + pub user_id: ID, + pub region: String, + pub instance_type: String, + pub storage_gb: i64, + pub ram_gb: i64, +} +#[handler] +pub fn CreateCluster(input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { + let data: CreateClusterInput = match sonic_rs::from_slice(&input.request.body) { + Ok(data) => data, + Err(err) => return Err(GraphError::from(err)), + }; + + let mut remapping_vals = RemappingMap::new(); + let db = Arc::clone(&input.graph.storage); + let mut txn = db.graph_env.write_txn().unwrap(); + let user = G::new(Arc::clone(&db), &txn) + .n_from_id(&data.user_id) + .collect_to::>(); + let new_cluster = G::new_mut(Arc::clone(&db), &mut txn) +.add_n("Cluster", Some(props! { "created_at" => chrono::Utc::now().to_rfc3339(), "region" => data.region.clone(), "updated_at" => chrono::Utc::now().to_rfc3339() }), None).collect_to::>(); + let new_instance = G::new_mut(Arc::clone(&db), &mut txn) +.add_n("Instance", Some(props! { "region" => data.region.clone(), "ram_gb" => data.ram_gb.clone(), "created_at" => chrono::Utc::now().to_rfc3339(), "storage_gb" => data.storage_gb.clone(), "updated_at" => chrono::Utc::now().to_rfc3339(), "instance_type" => data.instance_type.clone() }), None).collect_to::>(); + G::new_mut(Arc::clone(&db), &mut txn) + .add_e( + "CreatedCluster", + None, + user.id(), + new_cluster.id(), + true, + EdgeType::Node, + ) + .collect_to::>(); + G::new_mut(Arc::clone(&db), &mut txn) + .add_e( + "CreatedInstance", + None, + new_cluster.id(), + new_instance.id(), + true, + EdgeType::Node, + ) + .collect_to::>(); + let mut return_vals: HashMap = HashMap::new(); + return_vals.insert( + "new_cluster".to_string(), + ReturnValue::from_traversal_value_array_with_mixin( + new_cluster.clone(), + remapping_vals.borrow_mut(), + ), + ); + + txn.commit().unwrap(); + response.body = sonic_rs::to_vec(&return_vals).unwrap(); + Ok(()) +} From 6347009bb9de1536714bcc812b65e8705d72a7e7 Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Thu, 10 Jul 2025 23:08:32 +0100 Subject: [PATCH 2/9] mvp working --- get_routes/src/lib.rs | 17 +++++--- helix-container/src/main.rs | 39 +++++++++++-------- helixdb/src/helix_gateway/router/dynamic.rs | 43 +++++++++++++++------ helixdb/src/helix_gateway/router/mod.rs | 2 +- helixdb/src/helix_gateway/router/router.rs | 7 ++++ query-container/Cargo.toml | 3 ++ query-container/src/lib.rs | 41 +++++++------------- 7 files changed, 89 insertions(+), 63 deletions(-) diff --git a/get_routes/src/lib.rs b/get_routes/src/lib.rs index df720920..9b0e1876 100644 --- a/get_routes/src/lib.rs +++ b/get_routes/src/lib.rs @@ -13,7 +13,10 @@ pub fn handler(_attr: TokenStream, item: TokenStream) -> TokenStream { let fn_name_str = fn_name.to_string(); println!("fn_name_str: {}", fn_name_str); // Create a unique static name for each handler - let static_name = quote::format_ident!("__HANDLER_REGISTRATION_{}", fn_name.to_string().to_uppercase()); + let static_name = quote::format_ident!( + "__HANDLER_REGISTRATION_{}", + fn_name.to_string().to_uppercase() + ); let expanded = quote! { #input_fn @@ -34,7 +37,6 @@ pub fn handler(_attr: TokenStream, item: TokenStream) -> TokenStream { expanded.into() } - #[proc_macro_attribute] pub fn local_handler(_attr: TokenStream, item: TokenStream) -> TokenStream { let input_fn = parse_macro_input!(item as ItemFn); @@ -42,7 +44,10 @@ pub fn local_handler(_attr: TokenStream, item: TokenStream) -> TokenStream { let fn_name_str = fn_name.to_string(); println!("fn_name_str: {}", fn_name_str); // Create a unique static name for each handler - let static_name = quote::format_ident!("__HANDLER_REGISTRATION_{}", fn_name.to_string().to_uppercase()); + let static_name = quote::format_ident!( + "__HANDLER_REGISTRATION_{}", + fn_name.to_string().to_uppercase() + ); let expanded = quote! { #input_fn @@ -69,7 +74,10 @@ pub fn mcp_handler(_attr: TokenStream, item: TokenStream) -> TokenStream { let fn_name = &input_fn.sig.ident; let fn_name_str = fn_name.to_string(); // Create a unique static name for each handler - let static_name = quote::format_ident!("__HANDLER_REGISTRATION_{}", fn_name.to_string().to_uppercase()); + let static_name = quote::format_ident!( + "__HANDLER_REGISTRATION_{}", + fn_name.to_string().to_uppercase() + ); let expanded = quote! { #input_fn @@ -89,4 +97,3 @@ pub fn mcp_handler(_attr: TokenStream, item: TokenStream) -> TokenStream { }; expanded.into() } - diff --git a/helix-container/src/main.rs b/helix-container/src/main.rs index 7a17c772..72529cf1 100644 --- a/helix-container/src/main.rs +++ b/helix-container/src/main.rs @@ -1,6 +1,7 @@ use helixdb::helix_engine::graph_core::config::Config; use helixdb::helix_engine::graph_core::graph_core::{HelixGraphEngine, HelixGraphEngineOpts}; use helixdb::helix_gateway::mcp::mcp::{MCPHandlerFn, MCPHandlerSubmission}; +use helixdb::helix_gateway::router::dynamic::Plugin; use helixdb::helix_gateway::{ gateway::{GatewayOpts, HelixGateway}, router::router::{HandlerFn, HandlerSubmission}, @@ -54,23 +55,27 @@ async fn main() { let submissions: Vec<_> = inventory::iter::.into_iter().collect(); println!("Found {} submissions", submissions.len()); - let routes = HashMap::from_iter( - submissions - .into_iter() - .map(|submission| { - println!("Processing submission for handler: {}", submission.0.name); - let handler = &submission.0; - let func: HandlerFn = Arc::new(handler.func); - ( - ( - "post".to_ascii_uppercase().to_string(), - format!("/{}", handler.name.to_string()), - ), - func, - ) - }) - .collect::>(), - ); + let routes = unsafe { Plugin::open("../target/release/libquery_container.dylib").unwrap() } + .get_queries() + .unwrap(); + + // let routes = HashMap::from_iter( + // submissions + // .into_iter() + // .map(|submission| { + // println!("Processing submission for handler: {}", submission.0.name); + // let handler = &submission.0; + // let func: HandlerFn = Arc::new(handler.func); + // ( + // ( + // "post".to_ascii_uppercase().to_string(), + // format!("/{}", handler.name.to_string()), + // ), + // func, + // ) + // }) + // .collect::>(), + // ); let mcp_submissions: Vec<_> = inventory::iter:: .into_iter() diff --git a/helixdb/src/helix_gateway/router/dynamic.rs b/helixdb/src/helix_gateway/router/dynamic.rs index d1ae256d..855cdb06 100644 --- a/helixdb/src/helix_gateway/router/dynamic.rs +++ b/helixdb/src/helix_gateway/router/dynamic.rs @@ -1,10 +1,16 @@ use libloading::{self, Library, Symbol}; -use std::{error::Error, ops::Deref, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, + error::Error, + ops::Deref, + path::{Path, PathBuf}, + sync::Arc, +}; use crate::{ helix_engine::types::GraphError, helix_gateway::router::{ - router::{HandlerInput, HelixRouter}, + router::{HandlerFn, HandlerInput, HelixRouter}, QueryHandler, }, protocol::response::Response, @@ -23,27 +29,40 @@ impl QueryHandler for DynHandler { } } -// impl QueryHandler for DynHandler { -// fn handle(&self, input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { -// self.func. -// } -// } - type DynQueryFn = extern "Rust" fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; type GetQueryFn = extern "Rust" fn() -> Vec<(String, DynQueryFn)>; -struct Plugin { +pub struct Plugin { lib: Arc, } impl Plugin { /// SAFETY: This must be called with a path to Helix query dynamic library, compiled with the same version of Rust as the main database - unsafe fn open(lib_path: PathBuf) -> Result> { - let lib = Library::new(lib_path)?; + pub unsafe fn open(lib_path: impl AsRef) -> Result> { + let lib = Library::new(lib_path.as_ref())?; Ok(Plugin { lib: Arc::new(lib) }) } - fn add_queries(&self, router: &mut HelixRouter) -> Result<(), Box> { + pub fn get_queries(&self) -> Result, Box> { + // SAFETY: If a valid file was opened it will have a get_queries function of this type + let get_fn: Symbol = unsafe { self.lib.get(b"get_queries")? }; + + let queries = get_fn(); + + let mut acc: HashMap<(String, String), HandlerFn> = HashMap::new(); + + for (n, func) in queries.into_iter() { + let handler = DynHandler { + _source: self.lib.clone(), + func, + }; + + acc.insert(("post".to_string(), format!("/{n}")), Arc::new(handler)); + } + Ok(acc) + } + + pub fn add_queries(&self, router: &mut HelixRouter) -> Result<(), Box> { // SAFETY: If a valid file was opened it will have a get_queries function of this type let get_fn: Symbol = unsafe { self.lib.get(b"get_queries")? }; diff --git a/helixdb/src/helix_gateway/router/mod.rs b/helixdb/src/helix_gateway/router/mod.rs index 1d2f940e..80c3c3f4 100644 --- a/helixdb/src/helix_gateway/router/mod.rs +++ b/helixdb/src/helix_gateway/router/mod.rs @@ -4,7 +4,7 @@ use crate::{ protocol::response::Response, }; -mod dynamic; +pub mod dynamic; pub mod router; pub trait QueryHandler: Send + Sync { diff --git a/helixdb/src/helix_gateway/router/router.rs b/helixdb/src/helix_gateway/router/router.rs index c401f8e3..50ca5c16 100644 --- a/helixdb/src/helix_gateway/router/router.rs +++ b/helixdb/src/helix_gateway/router/router.rs @@ -47,6 +47,13 @@ impl Handler { inventory::collect!(HandlerSubmission); +impl HandlerSubmission { + pub fn collect_linked_handlers() -> Vec<&'static HandlerSubmission> { + let submissions: Vec<_> = inventory::iter::.into_iter().collect(); + submissions + } +} + /// Router for handling requests and MCP requests /// /// Standard Routes and MCP Routes are stored in a HashMap with the method and path as the key diff --git a/query-container/Cargo.toml b/query-container/Cargo.toml index d543654b..b66ec9e0 100644 --- a/query-container/Cargo.toml +++ b/query-container/Cargo.toml @@ -3,6 +3,9 @@ name = "query-container" version = "0.1.0" edition = "2024" +[lib] +crate-type = ["cdylib"] + [dependencies] helixdb = { path = "../helixdb" } get_routes = { path = "../get_routes" } diff --git a/query-container/src/lib.rs b/query-container/src/lib.rs index 62c4cf43..7be8b6b2 100644 --- a/query-container/src/lib.rs +++ b/query-container/src/lib.rs @@ -1,8 +1,6 @@ -use std::sync::Arc; - use helixdb::{ helix_engine::types::GraphError, - helix_gateway::router::{QueryHandler, router::HandlerInput}, + helix_gateway::router::router::{HandlerInput, HandlerSubmission}, protocol::response::Response, }; @@ -10,32 +8,19 @@ mod query; type DynQueryFn = fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; -// basic type for function pointer -pub type BasicHandlerFn = fn(&HandlerInput, &mut Response) -> Result<(), GraphError>; - -// thread safe type for multi threaded use -pub type HandlerFn = Arc; - -#[derive(Clone, Debug)] -pub struct HandlerSubmission(pub Handler); - -#[derive(Clone, Debug)] -pub struct Handler { - pub name: &'static str, - pub func: BasicHandlerFn, -} - -impl Handler { - pub const fn new(name: &'static str, func: BasicHandlerFn) -> Self { - Self { name, func } - } -} - -inventory::collect!(HandlerSubmission); - #[unsafe(no_mangle)] pub extern "Rust" fn get_queries() -> Vec<(String, DynQueryFn)> { - let submissions: Vec<_> = inventory::iter::.into_iter().collect(); + println!("get_queries called!!!!\n\n\n"); + let submissions = HandlerSubmission::collect_linked_handlers() + .into_iter() + .collect::>(); + + println!("got {} submissions", submissions.len()); + + let ret = submissions + .into_iter() + .map(|hs| (hs.0.name.to_owned(), hs.0.func)) + .collect(); - todo!() + ret } From ed9b2f572ca015acbb1e5ec1d4610002ce725f6b Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Mon, 14 Jul 2025 13:56:21 +0100 Subject: [PATCH 3/9] working --- helix-container/src/main.rs | 40 +++++++++---------- helixdb/src/helix_gateway/router/dynamic.rs | 2 +- helixdb/src/helix_gateway/router/router.rs | 31 ++++++++++---- .../helix_gateway/thread_pool/thread_pool.rs | 24 ++++++----- 4 files changed, 60 insertions(+), 37 deletions(-) diff --git a/helix-container/src/main.rs b/helix-container/src/main.rs index 72529cf1..fd95088c 100644 --- a/helix-container/src/main.rs +++ b/helix-container/src/main.rs @@ -55,27 +55,27 @@ async fn main() { let submissions: Vec<_> = inventory::iter::.into_iter().collect(); println!("Found {} submissions", submissions.len()); - let routes = unsafe { Plugin::open("../target/release/libquery_container.dylib").unwrap() } - .get_queries() - .unwrap(); + // let routes = unsafe { Plugin::open("../target/release/libquery_container.dylib").unwrap() } + // .get_queries() + // .unwrap(); - // let routes = HashMap::from_iter( - // submissions - // .into_iter() - // .map(|submission| { - // println!("Processing submission for handler: {}", submission.0.name); - // let handler = &submission.0; - // let func: HandlerFn = Arc::new(handler.func); - // ( - // ( - // "post".to_ascii_uppercase().to_string(), - // format!("/{}", handler.name.to_string()), - // ), - // func, - // ) - // }) - // .collect::>(), - // ); + let routes = HashMap::from_iter( + submissions + .into_iter() + .map(|submission| { + println!("Processing submission for handler: {}", submission.0.name); + let handler = &submission.0; + let func: HandlerFn = Arc::new(handler.func); + ( + ( + "post".to_ascii_uppercase().to_string(), + format!("/{}", handler.name.to_string()), + ), + func, + ) + }) + .collect::>(), + ); let mcp_submissions: Vec<_> = inventory::iter:: .into_iter() diff --git a/helixdb/src/helix_gateway/router/dynamic.rs b/helixdb/src/helix_gateway/router/dynamic.rs index 855cdb06..17c14abc 100644 --- a/helixdb/src/helix_gateway/router/dynamic.rs +++ b/helixdb/src/helix_gateway/router/dynamic.rs @@ -57,7 +57,7 @@ impl Plugin { func, }; - acc.insert(("post".to_string(), format!("/{n}")), Arc::new(handler)); + acc.insert(("POST".to_string(), format!("/{n}")), Arc::new(handler)); } Ok(acc) } diff --git a/helixdb/src/helix_gateway/router/router.rs b/helixdb/src/helix_gateway/router/router.rs index 50ca5c16..e6809118 100644 --- a/helixdb/src/helix_gateway/router/router.rs +++ b/helixdb/src/helix_gateway/router/router.rs @@ -7,15 +7,17 @@ // returns response +use tokio::sync::RwLock; + use crate::{ helix_engine::{graph_core::graph_core::HelixGraphEngine, types::GraphError}, helix_gateway::{ mcp::mcp::{MCPHandlerFn, MCPToolInput}, - router::QueryHandler, + router::{dynamic::Plugin, QueryHandler}, }, }; use core::fmt; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, ffi::OsStr, os::unix::ffi::OsStrExt, sync::Arc}; use crate::protocol::{request::Request, response::Response}; @@ -59,7 +61,7 @@ impl HandlerSubmission { /// Standard Routes and MCP Routes are stored in a HashMap with the method and path as the key pub struct HelixRouter { /// Method+Path => Function - pub routes: HashMap<(String, String), HandlerFn>, + pub routes: RwLock>, pub mcp_routes: HashMap<(String, String), MCPHandlerFn>, } @@ -78,14 +80,16 @@ impl HelixRouter { None => HashMap::new(), }; Self { - routes: rts, + routes: RwLock::new(rts), mcp_routes: mcp_rts, } } /// Add a route to the router - pub fn add_route(&mut self, method: &str, path: &str, handler: HandlerFn) { + pub async fn add_route(&mut self, method: &str, path: &str, handler: HandlerFn) { self.routes + .write() + .await .insert((method.to_uppercase(), path.to_string()), handler); } @@ -101,7 +105,7 @@ impl HelixRouter { /// /// * `Ok(())` if the request was handled successfully /// * `Err(RouterError)` if there was an error handling the request - pub fn handle( + pub async fn handle( &self, graph_access: Arc, request: Request, @@ -109,7 +113,20 @@ impl HelixRouter { ) -> Result<(), GraphError> { let route_key = (request.method.clone(), request.path.clone()); - if let Some(handler) = self.routes.get(&route_key) { + if route_key.0 == "PATCH" { + let body = OsStr::from_bytes(&request.body); + + let plugin = unsafe { Plugin::open(body) }.unwrap(); + + let qs = plugin.get_queries().unwrap(); + + self.routes.write().await.extend(qs); + + response.status = 200; + return Ok(()); + } + + if let Some(handler) = self.routes.read().await.get(&route_key) { let input = HandlerInput { request, graph: Arc::clone(&graph_access), diff --git a/helixdb/src/helix_gateway/thread_pool/thread_pool.rs b/helixdb/src/helix_gateway/thread_pool/thread_pool.rs index bb389992..53cc3b22 100644 --- a/helixdb/src/helix_gateway/thread_pool/thread_pool.rs +++ b/helixdb/src/helix_gateway/thread_pool/thread_pool.rs @@ -7,17 +7,16 @@ use crate::helix_gateway::router::router::{HelixRouter, RouterError}; use crate::protocol::request::Request; use crate::protocol::response::Response; - extern crate tokio; use tokio::net::TcpStream; /// Worker for handling requests -/// +/// /// A worker is a thread that handles requests -/// +/// /// It receives a connection from the thread pool and handles the request -/// +/// /// It sends the response back to the client pub struct Worker { pub id: usize, @@ -26,7 +25,7 @@ pub struct Worker { impl Worker { /// Creates a new worker - /// + /// /// It receives a connection from the thread pool and handles the request /// It sends the response back to the client fn new( @@ -54,7 +53,10 @@ impl Worker { }; let mut response = Response::new(); - if let Err(e) = router.handle(Arc::clone(&graph_access), request, &mut response) { + if let Err(e) = router + .handle(Arc::clone(&graph_access), request, &mut response) + .await + { eprintln!("Error handling request: {:?}", e); response.status = 500; response.body = format!("\n{:?}", e).into_bytes(); @@ -104,11 +106,15 @@ impl ThreadPool { let (tx, rx) = flume::bounded::(1000); // TODO: make this configurable let mut workers = Vec::with_capacity(size); for id in 0..size { - workers.push(Worker::new(id, Arc::clone(&graph), Arc::clone(&router), rx.clone())); + workers.push(Worker::new( + id, + Arc::clone(&graph), + Arc::clone(&router), + rx.clone(), + )); } println!("Thread pool initialized with {} workers", workers.len()); - Ok(ThreadPool { sender: tx, num_unused_workers: Mutex::new(size), @@ -116,4 +122,4 @@ impl ThreadPool { workers, }) } -} \ No newline at end of file +} From 8a29ad605ad237009032e80cc13bff879164db2c Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Tue, 15 Jul 2025 16:46:16 +0100 Subject: [PATCH 4/9] wip --- helix-cli/src/main.rs | 4 +- query-container/src/query.rs | 244 ----------------------------------- 2 files changed, 2 insertions(+), 246 deletions(-) diff --git a/helix-cli/src/main.rs b/helix-cli/src/main.rs index 817d509c..e1f5839f 100644 --- a/helix-cli/src/main.rs +++ b/helix-cli/src/main.rs @@ -559,11 +559,11 @@ fi let output = dirs::home_dir() .map(|path| { - path.join(".helix/repo/helix-db/helix-container") + path.join(".helix/repo/helix-db/query-container") .to_string_lossy() .into_owned() }) - .unwrap_or_else(|| "./.helix/repo/helix-db/helix-container".to_string()); + .unwrap_or_else(|| "./.helix/repo/helix-db/query-container".to_string()); let files = match check_and_read_files(&path) { Ok(files) if !files.is_empty() => files, diff --git a/query-container/src/query.rs b/query-container/src/query.rs index 883c605b..8b137891 100644 --- a/query-container/src/query.rs +++ b/query-container/src/query.rs @@ -1,245 +1 @@ -use chrono::{DateTime, Utc}; -use get_routes::handler; -use heed3::RoTxn; -use helixdb::helix_engine::vector_core::vector::HVector; -use helixdb::{ - exclude_field, field_remapping, identifier_remapping, traversal_remapping, value_remapping, -}; -use helixdb::{ - helix_engine::graph_core::ops::{ - bm25::search_bm25::SearchBM25Adapter, - g::G, - in_::{in_::InAdapter, in_e::InEdgesAdapter, to_n::ToNAdapter, to_v::ToVAdapter}, - out::{ - from_n::FromNAdapter, from_v::FromVAdapter, out::OutAdapter, out_e::OutEdgesAdapter, - }, - source::{ - add_e::{AddEAdapter, EdgeType}, - add_n::AddNAdapter, - e_from_id::EFromIdAdapter, - e_from_type::EFromTypeAdapter, - n_from_id::NFromIdAdapter, - n_from_index::NFromIndexAdapter, - n_from_type::NFromTypeAdapter, - }, - tr_val::{Traversable, TraversalVal}, - util::{ - dedup::DedupAdapter, drop::Drop, filter_mut::FilterMut, filter_ref::FilterRefAdapter, - map::MapAdapter, paths::ShortestPathAdapter, props::PropsAdapter, range::RangeAdapter, - update::UpdateAdapter, - }, - vectors::{ - brute_force_search::BruteForceSearchVAdapter, insert::InsertVAdapter, - search::SearchVAdapter, - }, - }, - helix_engine::types::GraphError, - helix_gateway::router::router::HandlerInput, - node_matches, props, - protocol::count::Count, - protocol::remapping::{RemappingMap, ResponseRemapping}, - protocol::response::Response, - protocol::{ - filterable::Filterable, id::ID, remapping::Remapping, return_values::ReturnValue, - value::Value, - }, -}; -use sonic_rs::{Deserialize, Serialize}; -use std::cell::RefCell; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::time::Instant; -pub struct Cluster { - pub region: String, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -pub struct Instance { - pub region: String, - pub instance_type: String, - pub storage_gb: i64, - pub ram_gb: i64, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -pub struct User { - pub gh_id: u64, - pub gh_login: String, - pub name: String, - pub email: String, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -pub struct CreatedCluster { - pub from: User, - pub to: Cluster, -} - -pub struct CreatedInstance { - pub from: Cluster, - pub to: Instance, -} - -#[derive(Serialize, Deserialize)] -pub struct GetInstancesForUserInput { - pub user_id: ID, -} - -#[handler] -pub fn GetInstancesForUser( - input: &HandlerInput, - response: &mut Response, -) -> Result<(), GraphError> { - let data: GetInstancesForUserInput = match sonic_rs::from_slice(&input.request.body) { - Ok(data) => data, - Err(err) => return Err(GraphError::from(err)), - }; - - let mut remapping_vals = RemappingMap::new(); - let db = Arc::clone(&input.graph.storage); - let txn = db.graph_env.read_txn().unwrap(); - let instances = G::new(Arc::clone(&db), &txn) - .n_from_id(&data.user_id) - .out("CreatedCluster", &EdgeType::Node) - .out("CreatedInstance", &EdgeType::Node) - .collect_to::>(); - let mut return_vals: HashMap = HashMap::new(); - return_vals.insert( - "instances".to_string(), - ReturnValue::from_traversal_value_array_with_mixin( - instances.clone(), - remapping_vals.borrow_mut(), - ), - ); - - txn.commit().unwrap(); - response.body = sonic_rs::to_vec(&return_vals).unwrap(); - Ok(()) -} - -#[derive(Serialize, Deserialize)] -pub struct CreateUserInput { - pub gh_id: u64, - pub gh_login: String, - pub name: String, - pub email: String, -} -#[handler] -pub fn CreateUser(input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { - let data: CreateUserInput = match sonic_rs::from_slice(&input.request.body) { - Ok(data) => data, - Err(err) => return Err(GraphError::from(err)), - }; - - let mut remapping_vals = RemappingMap::new(); - let db = Arc::clone(&input.graph.storage); - let mut txn = db.graph_env.write_txn().unwrap(); - let user = G::new_mut(Arc::clone(&db), &mut txn) -.add_n("User", Some(props! { "gh_id" => data.gh_id.clone(), "gh_login" => data.gh_login.clone(), "email" => data.email.clone(), "created_at" => chrono::Utc::now().to_rfc3339(), "updated_at" => chrono::Utc::now().to_rfc3339(), "name" => data.name.clone() }), Some(&["gh_id"])).collect_to::>(); - let mut return_vals: HashMap = HashMap::new(); - return_vals.insert( - "user".to_string(), - ReturnValue::from_traversal_value_array_with_mixin( - user.clone(), - remapping_vals.borrow_mut(), - ), - ); - - txn.commit().unwrap(); - response.body = sonic_rs::to_vec(&return_vals).unwrap(); - Ok(()) -} - -#[derive(Serialize, Deserialize)] -pub struct LookupUserInput { - pub gh_id: u64, -} -#[handler] -pub fn LookupUser(input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { - let data: LookupUserInput = match sonic_rs::from_slice(&input.request.body) { - Ok(data) => data, - Err(err) => return Err(GraphError::from(err)), - }; - - let mut remapping_vals = RemappingMap::new(); - let db = Arc::clone(&input.graph.storage); - let txn = db.graph_env.read_txn().unwrap(); - let user = G::new(Arc::clone(&db), &txn) - .n_from_index("gh_id", &data.gh_id) - .collect_to::>(); - let mut return_vals: HashMap = HashMap::new(); - return_vals.insert( - "user".to_string(), - ReturnValue::from_traversal_value_array_with_mixin( - user.clone(), - remapping_vals.borrow_mut(), - ), - ); - - txn.commit().unwrap(); - response.body = sonic_rs::to_vec(&return_vals).unwrap(); - Ok(()) -} - -#[derive(Serialize, Deserialize)] -pub struct CreateClusterInput { - pub user_id: ID, - pub region: String, - pub instance_type: String, - pub storage_gb: i64, - pub ram_gb: i64, -} -#[handler] -pub fn CreateCluster(input: &HandlerInput, response: &mut Response) -> Result<(), GraphError> { - let data: CreateClusterInput = match sonic_rs::from_slice(&input.request.body) { - Ok(data) => data, - Err(err) => return Err(GraphError::from(err)), - }; - - let mut remapping_vals = RemappingMap::new(); - let db = Arc::clone(&input.graph.storage); - let mut txn = db.graph_env.write_txn().unwrap(); - let user = G::new(Arc::clone(&db), &txn) - .n_from_id(&data.user_id) - .collect_to::>(); - let new_cluster = G::new_mut(Arc::clone(&db), &mut txn) -.add_n("Cluster", Some(props! { "created_at" => chrono::Utc::now().to_rfc3339(), "region" => data.region.clone(), "updated_at" => chrono::Utc::now().to_rfc3339() }), None).collect_to::>(); - let new_instance = G::new_mut(Arc::clone(&db), &mut txn) -.add_n("Instance", Some(props! { "region" => data.region.clone(), "ram_gb" => data.ram_gb.clone(), "created_at" => chrono::Utc::now().to_rfc3339(), "storage_gb" => data.storage_gb.clone(), "updated_at" => chrono::Utc::now().to_rfc3339(), "instance_type" => data.instance_type.clone() }), None).collect_to::>(); - G::new_mut(Arc::clone(&db), &mut txn) - .add_e( - "CreatedCluster", - None, - user.id(), - new_cluster.id(), - true, - EdgeType::Node, - ) - .collect_to::>(); - G::new_mut(Arc::clone(&db), &mut txn) - .add_e( - "CreatedInstance", - None, - new_cluster.id(), - new_instance.id(), - true, - EdgeType::Node, - ) - .collect_to::>(); - let mut return_vals: HashMap = HashMap::new(); - return_vals.insert( - "new_cluster".to_string(), - ReturnValue::from_traversal_value_array_with_mixin( - new_cluster.clone(), - remapping_vals.borrow_mut(), - ), - ); - - txn.commit().unwrap(); - response.body = sonic_rs::to_vec(&return_vals).unwrap(); - Ok(()) -} From 52aa238f71cfd8655433335bebf21da3fc27ff20 Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Tue, 15 Jul 2025 18:26:45 +0100 Subject: [PATCH 5/9] switch extend to replace --- helix-container/src/main.rs | 5 ----- helixdb/src/helix_gateway/router/dynamic.rs | 17 ----------------- helixdb/src/helix_gateway/router/router.rs | 2 +- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/helix-container/src/main.rs b/helix-container/src/main.rs index fd95088c..7a17c772 100644 --- a/helix-container/src/main.rs +++ b/helix-container/src/main.rs @@ -1,7 +1,6 @@ use helixdb::helix_engine::graph_core::config::Config; use helixdb::helix_engine::graph_core::graph_core::{HelixGraphEngine, HelixGraphEngineOpts}; use helixdb::helix_gateway::mcp::mcp::{MCPHandlerFn, MCPHandlerSubmission}; -use helixdb::helix_gateway::router::dynamic::Plugin; use helixdb::helix_gateway::{ gateway::{GatewayOpts, HelixGateway}, router::router::{HandlerFn, HandlerSubmission}, @@ -55,10 +54,6 @@ async fn main() { let submissions: Vec<_> = inventory::iter::.into_iter().collect(); println!("Found {} submissions", submissions.len()); - // let routes = unsafe { Plugin::open("../target/release/libquery_container.dylib").unwrap() } - // .get_queries() - // .unwrap(); - let routes = HashMap::from_iter( submissions .into_iter() diff --git a/helixdb/src/helix_gateway/router/dynamic.rs b/helixdb/src/helix_gateway/router/dynamic.rs index 17c14abc..15753020 100644 --- a/helixdb/src/helix_gateway/router/dynamic.rs +++ b/helixdb/src/helix_gateway/router/dynamic.rs @@ -61,21 +61,4 @@ impl Plugin { } Ok(acc) } - - pub fn add_queries(&self, router: &mut HelixRouter) -> Result<(), Box> { - // SAFETY: If a valid file was opened it will have a get_queries function of this type - let get_fn: Symbol = unsafe { self.lib.get(b"get_queries")? }; - - let queries = get_fn(); - - for (name, func) in queries { - let handler = DynHandler { - _source: self.lib.clone(), - func, - }; - router.add_route("post", &format!("/{name}"), Arc::new(handler)); - } - - todo!() - } } diff --git a/helixdb/src/helix_gateway/router/router.rs b/helixdb/src/helix_gateway/router/router.rs index e6809118..fe223bcd 100644 --- a/helixdb/src/helix_gateway/router/router.rs +++ b/helixdb/src/helix_gateway/router/router.rs @@ -120,7 +120,7 @@ impl HelixRouter { let qs = plugin.get_queries().unwrap(); - self.routes.write().await.extend(qs); + *self.routes.write().await = qs; response.status = 200; return Ok(()); From fcd5f2a95f165a81b840d31f015b61ce13009f79 Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Thu, 17 Jul 2025 10:43:35 +0100 Subject: [PATCH 6/9] fix --- helix-cli/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helix-cli/src/main.rs b/helix-cli/src/main.rs index e1f5839f..b9ff800c 100644 --- a/helix-cli/src/main.rs +++ b/helix-cli/src/main.rs @@ -15,7 +15,7 @@ use std::{ fmt::Write, fs, io::Write as iWrite, - path::{Path, PathBuf}, + path::PathBuf, process::{Command, Stdio}, }; @@ -559,11 +559,11 @@ fi let output = dirs::home_dir() .map(|path| { - path.join(".helix/repo/helix-db/query-container") + path.join(".helix/repo/helix-db/helix-container") .to_string_lossy() .into_owned() }) - .unwrap_or_else(|| "./.helix/repo/helix-db/query-container".to_string()); + .unwrap_or_else(|| "./.helix/repo/helix-db/helix-container".to_string()); let files = match check_and_read_files(&path) { Ok(files) if !files.is_empty() => files, From 8649d7bf5ff8dab8b5f8eacc1702ff11c7f977be Mon Sep 17 00:00:00 2001 From: Raphael Darley Date: Sat, 19 Jul 2025 19:10:09 +0100 Subject: [PATCH 7/9] merge --- Cargo.lock | 1297 +---- Cargo.toml | 1 - hbuild_redeploy/Cargo.toml | 17 - hbuild_redeploy/src/main.rs | 296 - helix-cli/Cargo.toml | 3 +- helix-cli/src/main.rs | 109 +- helix-cli/src/utils.rs | 14 +- helix-container/src/main.rs | 12 +- helix-container/src/queries.rs | 6 + helix-db/Cargo.toml | 3 +- helix-db/src/grammar.pest | 20 +- helix-db/src/helix_engine/bm25/bm25.rs | 4 +- .../src/helix_engine/graph_core/config.rs | 126 +- .../src/helix_engine/graph_core/graph_core.rs | 2 +- .../src/helix_engine/graph_core/ops/tr_val.rs | 8 +- .../helix_engine/graph_core/ops/util/drop.rs | 13 +- .../helix_engine/graph_core/ops/util/props.rs | 41 +- .../helix_engine/graph_core/ops/util/range.rs | 29 +- .../graph_core/ops/vectors/insert.rs | 10 +- .../graph_core/ops/vectors/search.rs | 16 +- .../helix_engine/graph_core/traversal_iter.rs | 12 +- .../graph_core/traversal_tests.rs | 166 +- helix-db/src/helix_engine/macros.rs | 23 +- .../helix_engine/storage_core/storage_core.rs | 16 +- helix-db/src/helix_engine/types.rs | 2 + helix-db/src/helix_engine/vector_core/hnsw.rs | 12 + helix-db/src/helix_engine/vector_core/mod.rs | 3 +- .../src/helix_engine/vector_core/utils.rs | 145 + .../src/helix_engine/vector_core/vector.rs | 60 +- .../helix_engine/vector_core/vector_core.rs | 157 +- .../embedding_providers.rs | 2 +- helix-db/src/helix_gateway/mcp/mcp.rs | 2 +- helix-db/src/helix_gateway/mcp/tools.rs | 38 +- helix-db/src/helix_gateway/mod.rs | 5 +- helix-db/src/helixc/analyzer/analyzer.rs | 4872 +---------------- .../src/helixc/analyzer/analyzer_tests.rs | 325 -- helix-db/src/helixc/analyzer/diagnostic.rs | 48 + helix-db/src/helixc/analyzer/error_codes.rs | 113 +- helix-db/src/helixc/analyzer/errors.rs | 67 + .../analyzer/methods/exclude_validation.rs | 131 + .../analyzer/methods/graph_step_validation.rs | 646 +++ .../analyzer/methods/infer_expr_type.rs | 1121 ++++ helix-db/src/helixc/analyzer/methods/mod.rs | 8 + .../analyzer/methods/object_validation.rs | 625 +++ .../analyzer/methods/query_validation.rs | 191 + .../helixc/analyzer/methods/schema_methods.rs | 154 + .../analyzer/methods/statement_validation.rs | 230 + .../analyzer/methods/traversal_validation.rs | 1102 ++++ helix-db/src/helixc/analyzer/mod.rs | 11 +- helix-db/src/helixc/analyzer/pretty.rs | 12 +- helix-db/src/helixc/analyzer/types.rs | 123 + helix-db/src/helixc/analyzer/utils.rs | 187 + .../src/helixc/generator/generator_types.rs | 9 +- .../generator/object_remapping_generation.rs | 50 +- helix-db/src/helixc/generator/source_steps.rs | 4 +- .../src/helixc/generator/traversal_steps.rs | 22 - helix-db/src/helixc/generator/utils.rs | 64 +- helix-db/src/helixc/parser/helix_parser.rs | 84 +- helix-db/src/protocol/format.rs | 120 + helix-db/src/protocol/mod.rs | 1 + helix-db/src/protocol/response.rs | 20 +- helix-db/src/protocol/return_values.rs | 7 +- helix-db/src/protocol/value.rs | 143 +- helix-db/src/utils/filterable.rs | 54 +- helix-macros/Cargo.toml | 3 +- helix-macros/src/lib.rs | 34 +- hql-tests/file1/config.hx.json | 6 +- hql-tests/file10/config.hx.json | 6 +- hql-tests/file11/config.hx.json | 6 +- hql-tests/file12/config.hx.json | 6 +- hql-tests/file13/config.hx.json | 6 +- hql-tests/file14/config.hx.json | 6 +- hql-tests/file15/config.hx.json | 6 +- hql-tests/file16/config.hx.json | 6 +- hql-tests/file17/config.hx.json | 6 +- hql-tests/file18/config.hx.json | 6 +- hql-tests/file19/config.hx.json | 6 +- hql-tests/file2/config.hx.json | 6 +- hql-tests/file20/config.hx.json | 6 +- hql-tests/file21/config.hx.json | 6 +- hql-tests/file22/config.hx.json | 7 +- hql-tests/file23/config.hx.json | 6 +- hql-tests/file24/config.hx.json | 6 +- hql-tests/file25/config.hx.json | 6 +- hql-tests/file26/config.hx.json | 6 +- hql-tests/file27/config.hx.json | 6 +- hql-tests/file28/config.hx.json | 6 +- hql-tests/file29/config.hx.json | 8 +- hql-tests/file3/config.hx.json | 6 +- hql-tests/file30/config.hx.json | 6 +- hql-tests/file31/config.hx.json | 6 +- hql-tests/file32/config.hx.json | 5 +- hql-tests/file33/config.hx.json | 6 +- hql-tests/file34/config.hx.json | 6 +- hql-tests/file35/config.hx.json | 6 +- hql-tests/file36/config.hx.json | 8 +- hql-tests/file37/config.hx.json | 6 +- hql-tests/file38/config.hx.json | 6 +- hql-tests/file39/config.hx.json | 6 +- hql-tests/file4/config.hx.json | 6 +- hql-tests/file40/config.hx.json | 6 +- hql-tests/file41/config.hx.json | 6 +- hql-tests/file42/config.hx.json | 6 +- hql-tests/file43/config.hx.json | 6 +- hql-tests/file43/file43.hx | 35 + hql-tests/file43/schema.hx | 6 + hql-tests/file44/config.hx.json | 6 +- hql-tests/file44/file44.hx | 397 ++ hql-tests/file44/schema.hx | 226 +- hql-tests/file45/config.hx.json | 6 +- hql-tests/file45/file45.hx | 10 + hql-tests/file46/config.hx.json | 6 +- hql-tests/file46/file46.hx | 19 + hql-tests/file46/schema.hx | 65 +- hql-tests/file47/config.hx.json | 6 +- hql-tests/file47/file47.hx | 15 + hql-tests/file47/schema.hx | 65 +- hql-tests/file48/config.hx.json | 6 +- hql-tests/file48/file48.hx | 3 + hql-tests/file48/schema.hx | 65 +- hql-tests/file49/config.hx.json | 6 +- hql-tests/file49/file49.hx | 11 + hql-tests/file5/config.hx.json | 6 +- hql-tests/file50/config.hx.json | 6 +- hql-tests/file50/file50.hx | 13 + hql-tests/file51/config.hx.json | 6 +- hql-tests/file52/config.hx.json | 6 +- hql-tests/file53/config.hx.json | 6 +- hql-tests/file54/config.hx.json | 6 +- hql-tests/file55/config.hx.json | 6 +- hql-tests/file56/config.hx.json | 6 +- hql-tests/file57/config.hx.json | 6 +- hql-tests/file58/config.hx.json | 6 +- hql-tests/file59/config.hx.json | 6 +- hql-tests/file6/config.hx.json | 6 +- hql-tests/file60/config.hx.json | 6 +- hql-tests/file61/config.hx.json | 6 +- hql-tests/file62/config.hx.json | 6 +- hql-tests/file63/config.hx.json | 6 +- hql-tests/file64/config.hx.json | 6 +- hql-tests/file65/config.hx.json | 6 +- hql-tests/file66/config.hx.json | 6 +- hql-tests/file67/config.hx.json | 6 +- hql-tests/file68/config.hx.json | 6 +- hql-tests/file69/config.hx.json | 6 +- hql-tests/file7/config.hx.json | 6 +- hql-tests/file70/config.hx.json | 6 +- hql-tests/file71/config.hx.json | 6 +- hql-tests/file72/config.hx.json | 6 +- hql-tests/file73/config.hx.json | 6 +- hql-tests/file74/config.hx.json | 6 +- hql-tests/file75/config.hx.json | 6 +- hql-tests/file76/config.hx.json | 6 +- hql-tests/file77/config.hx.json | 6 +- hql-tests/file78/config.hx.json | 6 +- hql-tests/file79/config.hx.json | 6 +- hql-tests/file8/config.hx.json | 6 +- hql-tests/file80/config.hx.json | 6 +- hql-tests/file81/config.hx.json | 6 +- hql-tests/file82/config.hx.json | 6 +- hql-tests/file83/config.hx.json | 6 +- hql-tests/file84/config.hx.json | 6 +- hql-tests/file85/config.hx.json | 6 +- hql-tests/file86/config.hx.json | 6 +- hql-tests/file87/config.hx.json | 6 +- hql-tests/file88/config.hx.json | 6 +- hql-tests/file89/config.hx.json | 6 +- hql-tests/file9/config.hx.json | 6 +- hql-tests/file90/config.hx.json | 6 +- hql-tests/file91/config.hx.json | 6 +- hql-tests/file92/config.hx.json | 6 +- hql-tests/file93/config.hx.json | 6 +- hql-tests/file94/config.hx.json | 6 +- hql-tests/file95/config.hx.json | 6 +- hql-tests/file96/config.hx.json | 6 +- hql-tests/file97/config.hx.json | 6 +- hql-tests/file98/config.hx.json | 6 +- hql-tests/file99/config.hx.json | 6 +- hql-tests/run.sh | 12 +- hql-tests/src/main.rs | 2 + 180 files changed, 7136 insertions(+), 7657 deletions(-) delete mode 100644 hbuild_redeploy/Cargo.toml delete mode 100644 hbuild_redeploy/src/main.rs create mode 100644 helix-db/src/helix_engine/vector_core/utils.rs delete mode 100644 helix-db/src/helixc/analyzer/analyzer_tests.rs create mode 100644 helix-db/src/helixc/analyzer/diagnostic.rs create mode 100644 helix-db/src/helixc/analyzer/errors.rs create mode 100644 helix-db/src/helixc/analyzer/methods/exclude_validation.rs create mode 100644 helix-db/src/helixc/analyzer/methods/graph_step_validation.rs create mode 100644 helix-db/src/helixc/analyzer/methods/infer_expr_type.rs create mode 100644 helix-db/src/helixc/analyzer/methods/mod.rs create mode 100644 helix-db/src/helixc/analyzer/methods/object_validation.rs create mode 100644 helix-db/src/helixc/analyzer/methods/query_validation.rs create mode 100644 helix-db/src/helixc/analyzer/methods/schema_methods.rs create mode 100644 helix-db/src/helixc/analyzer/methods/statement_validation.rs create mode 100644 helix-db/src/helixc/analyzer/methods/traversal_validation.rs create mode 100644 helix-db/src/helixc/analyzer/utils.rs create mode 100644 helix-db/src/protocol/format.rs diff --git a/Cargo.lock b/Cargo.lock index 6bb0c495..afeb0aa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arc-swap" @@ -225,500 +225,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "aws-config" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 1.3.1", - "ring", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - -[[package]] -name = "aws-lc-rs" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-runtime" -version = "1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "1.82.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "fastrand", - "hex", - "hmac", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "lru", - "once_cell", - "percent-encoding", - "regex-lite", - "sha2", - "tracing", - "url", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" -dependencies = [ - "aws-credential-types", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "crypto-bigint 0.5.5", - "form_urlencoded", - "hex", - "hmac", - "http 0.2.12", - "http 1.3.1", - "once_cell", - "p256", - "percent-encoding", - "ring", - "sha2", - "subtle", - "time", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.63.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "crc64fast-nvme", - "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-client" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2 0.4.8", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.6.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.5", - "hyper-util", - "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.25", - "rustls-native-certs 0.8.1", - "rustls-pki-types", - "tokio", - "tower 0.5.2", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-observability" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" -dependencies = [ - "aws-smithy-runtime-api", - "once_cell", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-client", - "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "http-body 1.0.1", - "once_cell", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.3.1", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-types" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "rustc_version", - "tracing", -] - -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "hyper 1.6.0", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower 0.5.2", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "backtrace" version = "0.3.74" @@ -734,40 +240,12 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "base16ct" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - -[[package]] -name = "base64ct" -version = "1.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" - [[package]] name = "bincode" version = "1.3.3" @@ -777,29 +255,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.8.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.104", - "which", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -908,16 +363,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "castaway" version = "0.2.3" @@ -944,15 +389,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -984,17 +420,6 @@ dependencies = [ "phf 0.12.1", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.5.40" @@ -1035,15 +460,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" -[[package]] -name = "cmake" -version = "0.1.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.3" @@ -1099,12 +515,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "core-foundation" version = "0.9.4" @@ -1140,30 +550,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.4.2" @@ -1173,15 +559,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crc64fast-nvme" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" -dependencies = [ - "crc", -] - [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1247,28 +624,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -1291,16 +646,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85d3cef41d236720ed453e102153a53e4cc3d2fde848c0078a50cf249e8e3e5b" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "deranged" version = "0.4.1" @@ -1318,7 +663,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1383,56 +727,18 @@ dependencies = [ "phf 0.11.3", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" -[[package]] -name = "ecdsa" -version = "0.14.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" -dependencies = [ - "der", - "elliptic-curve", - "rfc6979", - "signature", -] - [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" -[[package]] -name = "elliptic-curve" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" -dependencies = [ - "base16ct", - "crypto-bigint 0.4.9", - "der", - "digest", - "ff", - "generic-array", - "group", - "pkcs8", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "encode_unicode" version = "1.0.0" @@ -1512,16 +818,6 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "ff" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "flate2" version = "1.1.2" @@ -1590,12 +886,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.31" @@ -1733,36 +1023,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" -[[package]] -name = "group" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.8" @@ -1774,7 +1034,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", + "http", "indexmap", "slab", "tokio", @@ -1817,23 +1077,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hbuild_redploy" -version = "0.1.0" -dependencies = [ - "anyhow", - "aws-config", - "aws-sdk-s3", - "axum", - "serde", - "sonic-rs", - "tokio", - "tower 0.4.13", - "tower-http 0.5.2", - "tracing", - "tracing-subscriber", -] - [[package]] name = "heck" version = "0.4.1" @@ -1888,8 +1131,9 @@ dependencies = [ [[package]] name = "helix-cli" -version = "1.0.115" +version = "1.0.118" dependencies = [ + "anyhow", "chrono", "clap", "dirs 6.0.0", @@ -1933,7 +1177,7 @@ dependencies = [ [[package]] name = "helix-db" -version = "1.0.115" +version = "1.0.118" dependencies = [ "bincode", "chrono", @@ -1941,7 +1185,7 @@ dependencies = [ "heed3", "helix-macros", "inventory", - "itertools 0.14.0", + "itertools", "kdam", "lazy_static", "libloading", @@ -1956,6 +1200,7 @@ dependencies = [ "sonic-rs", "tempfile", "tokio", + "tokio-util", "twox-hash", "url", "uuid", @@ -1963,8 +1208,9 @@ dependencies = [ [[package]] name = "helix-macros" -version = "0.1.3" +version = "0.1.5" dependencies = [ + "dirs 5.0.1", "proc-macro2", "quote", "syn 2.0.104", @@ -1982,15 +1228,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - [[package]] name = "home" version = "0.5.11" @@ -2005,7 +1242,7 @@ name = "hql-tests" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.22.1", + "base64", "clap", "futures", "octocrab", @@ -2014,17 +1251,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.3.1" @@ -2036,17 +1262,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -2054,7 +1269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http", ] [[package]] @@ -2065,8 +1280,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -2074,37 +1289,7 @@ dependencies = [ name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "hyper" @@ -2115,11 +1300,10 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.8", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2127,22 +1311,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.5" @@ -2150,15 +1318,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.3.1", - "hyper 1.6.0", + "http", + "hyper", "hyper-util", "log", - "rustls 0.23.25", - "rustls-native-certs 0.8.1", + "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower-service", ] @@ -2168,7 +1336,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.6.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -2183,7 +1351,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-util", "native-tls", "tokio", @@ -2200,9 +1368,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "hyper 1.6.0", + "http", + "http-body", + "hyper", "pin-project-lite", "socket2", "tokio", @@ -2457,15 +1625,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2539,7 +1698,7 @@ version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64 0.22.1", + "base64", "js-sys", "pem", "ring", @@ -2564,12 +1723,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.172" @@ -2662,15 +1815,6 @@ version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.2", -] - [[package]] name = "lz4" version = "1.28.1" @@ -2696,22 +1840,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - [[package]] name = "memchr" version = "2.7.4" @@ -2733,12 +1861,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.5" @@ -2811,16 +1933,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "now" version = "0.1.3" @@ -2839,16 +1951,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -2932,18 +2034,18 @@ checksum = "86996964f8b721067b6ed238aa0ccee56ecad6ee5e714468aa567992d05d2b91" dependencies = [ "arc-swap", "async-trait", - "base64 0.22.1", + "base64", "bytes", "cfg-if", "chrono", "either", "futures", "futures-util", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.6.0", - "hyper-rustls 0.27.5", + "hyper", + "hyper-rustls", "hyper-timeout", "hyper-util", "jsonwebtoken", @@ -2957,8 +2059,8 @@ dependencies = [ "serde_urlencoded", "snafu", "tokio", - "tower 0.5.2", - "tower-http 0.6.6", + "tower", + "tower-http", "tracing", "url", "web-time", @@ -3031,29 +2133,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "outref" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - -[[package]] -name = "p256" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" -dependencies = [ - "ecdsa", - "elliptic-curve", - "sha2", -] - [[package]] name = "page_size" version = "0.6.0" @@ -3099,7 +2178,7 @@ version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ - "base64 0.22.1", + "base64", "serde", ] @@ -3246,16 +2325,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkg-config" version = "0.3.32" @@ -3541,7 +2610,7 @@ checksum = "cbb83218b0c216104f0076cd1a005128be078f958125f3d59b094ee73d78c18e" dependencies = [ "ahash", "argminmax", - "base64 0.22.1", + "base64", "bytemuck", "chrono", "chrono-tz", @@ -3578,7 +2647,7 @@ checksum = "5c60ee85535590a38db6c703a21be4cb25342e40f573f070d1e16f9d84a53ac7" dependencies = [ "ahash", "async-stream", - "base64 0.22.1", + "base64", "brotli", "bytemuck", "ethnum", @@ -3815,16 +2884,6 @@ dependencies = [ "zerocopy 0.7.35", ] -[[package]] -name = "prettyplease" -version = "0.2.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" -dependencies = [ - "proc-macro2", - "syn 2.0.104", -] - [[package]] name = "proc-macro2" version = "1.0.95" @@ -4110,12 +3169,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-lite" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -4134,18 +3187,18 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "encoding_rs", "futures-channel", "futures-core", "futures-util", - "h2 0.4.8", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.6.0", - "hyper-rustls 0.27.5", + "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "ipnet", @@ -4156,7 +3209,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.2.0", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -4164,7 +3217,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tower 0.5.2", + "tower", "tower-service", "url", "wasm-bindgen", @@ -4173,17 +3226,6 @@ dependencies = [ "windows-registry", ] -[[package]] -name = "rfc6979" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" -dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", -] - [[package]] name = "ring" version = "0.17.14" @@ -4233,21 +3275,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.37.28" @@ -4288,46 +3315,21 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ - "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -4340,15 +3342,6 @@ dependencies = [ "security-framework 3.2.0", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -4364,23 +3357,12 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4422,30 +3404,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "sec1" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - [[package]] name = "secrecy" version = "0.10.3" @@ -4491,12 +3449,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - [[package]] name = "serde" version = "1.0.219" @@ -4583,15 +3535,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shlex" version = "1.3.0" @@ -4607,16 +3550,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - [[package]] name = "simd-json" version = "0.14.3" @@ -4778,16 +3711,6 @@ dependencies = [ "strum", ] -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der", -] - [[package]] name = "sqlparser" version = "0.53.0" @@ -5038,15 +3961,6 @@ dependencies = [ "syn 2.0.104", ] -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" version = "0.3.41" @@ -5142,23 +4056,13 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.25", + "rustls", "tokio", ] @@ -5176,12 +4080,13 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -5228,17 +4133,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.2" @@ -5256,23 +4150,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower-http" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" -dependencies = [ - "bitflags 2.8.0", - "bytes", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "pin-project-lite", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower-http" version = "0.6.6" @@ -5282,11 +4159,11 @@ dependencies = [ "bitflags 2.8.0", "bytes", "futures-util", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "iri-string", "pin-project-lite", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -5334,32 +4211,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" -dependencies = [ - "nu-ansi-term", - "sharded-slab", - "smallvec", - "thread_local", - "tracing-core", - "tracing-log", ] [[package]] @@ -5376,7 +4227,7 @@ checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" dependencies = [ "bytes", "data-encoding", - "http 1.3.1", + "http", "httparse", "log", "rand 0.9.1", @@ -5460,12 +4311,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" @@ -5501,12 +4346,6 @@ dependencies = [ "rand 0.9.1", ] -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - [[package]] name = "value-trait" version = "0.10.1" @@ -5531,12 +4370,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - [[package]] name = "walkdir" version = "2.5.0" @@ -5679,18 +4512,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "winapi" version = "0.3.9" @@ -6223,12 +5044,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 43f8b1bb..96c1aad9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,6 @@ members = [ "helix-container", "helix-macros", "helix-cli", - "hbuild_redeploy", "query-container", "hql-tests", ] diff --git a/hbuild_redeploy/Cargo.toml b/hbuild_redeploy/Cargo.toml deleted file mode 100644 index cfa2c99b..00000000 --- a/hbuild_redeploy/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "hbuild_redploy" -version = "0.1.0" -edition = "2021" - -[dependencies] -aws-config = "1.1.1" -aws-sdk-s3 = "1.11.0" -tokio = { version = "1.44.2", features = ["full"] } -anyhow = "1.0" -serde = { version = "1.0", features = ["derive"] } -sonic-rs = "0.5.0" -axum = "0.7" -tower = "0.4" -tower-http = { version = "0.5", features = ["trace"] } -tracing = "0.1" -tracing-subscriber = "0.3" \ No newline at end of file diff --git a/hbuild_redeploy/src/main.rs b/hbuild_redeploy/src/main.rs deleted file mode 100644 index 1a8a61cb..00000000 --- a/hbuild_redeploy/src/main.rs +++ /dev/null @@ -1,296 +0,0 @@ -use anyhow::Result; -use aws_config::BehaviorVersion; -use aws_sdk_s3::Client; -use axum::{ - extract::State, - http::StatusCode, - response::Json, - routing::{get, post}, - Router, -}; -use sonic_rs::{Deserialize, Serialize}; -use std::fs::File; -use std::io::Write; -use std::{net::SocketAddr, process::Command}; -use tower::ServiceBuilder; -use tower_http::trace::TraceLayer; -use tracing_subscriber; - -// Constants for timeouts -//const SOCKET_TIMEOUT: Duration = Duration::from_secs(30); - -// make sure build is run in sudo mode - -#[derive(Debug, Deserialize, Serialize)] -pub struct HBuildDeployRequest { - user_id: String, - instance_id: String, - version: String, -} - -#[derive(Debug, Serialize)] -pub struct DeployResponse { - success: bool, - message: String, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, -} - -impl DeployResponse { - fn success(message: String) -> Self { - Self { - success: true, - message, - error: None, - } - } - - fn error(message: String, error: String) -> Self { - Self { - success: false, - message, - error: Some(error), - } - } -} - -// Shared application state -#[derive(Clone)] -pub struct AppState { - s3_client: Client, - user_id: String, - cluster_id: String, -} - -// Handler for the health check endpoint -async fn health_handler() -> Json { - Json(DeployResponse::success("Service is healthy".to_string())) -} - -// Handler for the /redeploy endpoint -async fn redeploy_handler(State(state): State) -> Result, StatusCode> { - tracing::info!("Received redeploy request"); - - // Move the deployment logic here - tokio::spawn(async move { - if let Err(e) = perform_deployment(&state).await { - tracing::error!("Deployment failed: {:?}", e); - } - }); - - Ok(Json(DeployResponse::success( - "Deployment initiated successfully".to_string(), - ))) -} - -async fn perform_deployment(state: &AppState) -> Result<(), AdminError> { - // rename old binary - let mv_result = Command::new("mv") - .arg("helix") - .arg("helix_old") - .output() - .map_err(|e| AdminError::CommandError("Failed to backup old binary".to_string(), e))?; - - if !mv_result.status.success() { - return Err(AdminError::CommandError( - "Failed to backup old binary".to_string(), - std::io::Error::new(std::io::ErrorKind::Other, "mv command failed"), - )); - } - - // pull binary from s3 - let response = state - .s3_client - .get_object() - .bucket("helix-build") - .key(format!( - "{}/{}/helix/latest", - state.user_id, state.cluster_id - )) - .send() - .await - .map_err(|e| AdminError::S3DownloadError("Failed to download binary from S3".to_string(), e))?; - - // create binary file or overwrite if it exists - let mut file = File::create("helix") - .map_err(|e| AdminError::FileError("Failed to create new binary file".to_string(), e))?; - - let body = response - .body - .collect() - .await - .map_err(|e| AdminError::FileError("Failed to collect S3 response body".to_string(), - std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))? - .to_vec(); - - file.write_all(&body) - .map_err(|e| AdminError::FileError("Failed to write binary file".to_string(), e))?; - - // set permissions - let chmod_result = Command::new("sudo") - .arg("chmod") - .arg("+x") - .arg("helix") - .output() - .map_err(|e| AdminError::CommandError("Failed to set binary permissions".to_string(), e))?; - - if !chmod_result.status.success() { - return Err(AdminError::CommandError( - "Failed to set binary permissions".to_string(), - std::io::Error::new(std::io::ErrorKind::Other, "chmod command failed"), - )); - } - - // restart systemd service - let restart_result = Command::new("sudo") - .arg("systemctl") - .arg("restart") - .arg("helix") - .output() - .map_err(|e| AdminError::CommandError("Failed to restart service".to_string(), e))?; - - if !restart_result.status.success() { - return Err(AdminError::CommandError( - "Failed to restart service".to_string(), - std::io::Error::new(std::io::ErrorKind::Other, "systemctl restart failed"), - )); - } - - // check if service is running - let status_result = Command::new("sudo") - .arg("systemctl") - .arg("status") - .arg("helix") - .output() - .map_err(|e| AdminError::CommandError("Failed to check service status".to_string(), e))?; - - // if not revert - if !status_result.status.success() { - tracing::warn!("Service failed to start, reverting to old binary"); - - let revert_result = Command::new("mv") - .arg("helix_old") - .arg("helix") - .output(); - - if let Err(e) = revert_result { - tracing::error!("Failed to revert binary: {:?}", e); - } - - let restart_old_result = Command::new("sudo") - .arg("systemctl") - .arg("restart") - .arg("helix") - .output(); - - if let Err(e) = restart_old_result { - tracing::error!("Failed to restart with old binary: {:?}", e); - } - - return Err(AdminError::CommandError( - "Service failed to start with new binary, reverted".to_string(), - std::io::Error::new(std::io::ErrorKind::Other, "Service startup failed"), - )); - } else { - // delete old binary - let rm_result = Command::new("rm") - .arg("helix_old") - .output(); - - if let Err(e) = rm_result { - tracing::warn!("Failed to delete old binary: {:?}", e); - } - - tracing::info!("Deployment completed successfully"); - } - - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<(), AdminError> { - // Initialize tracing - tracing_subscriber::fmt::init(); - - tracing::info!("Starting helix build service"); - - // Initialize AWS SDK with explicit region configuration - let bucket_region = std::env::var("S3_BUCKET_REGION").unwrap_or("us-west-1".to_string()); - tracing::info!("Using S3 bucket region: {}", bucket_region); - - let config = aws_config::load_defaults(BehaviorVersion::latest()) - .await - .to_builder() - .region(aws_config::Region::new(bucket_region.clone())) - .build(); - let s3_client = Client::new(&config); - - tracing::info!("AWS region configured: {:?}", config.region()); - - let user_id = std::env::var("USER_ID").expect("USER_ID is not set"); - let cluster_id = std::env::var("CLUSTER_ID").expect("CLUSTER_ID is not set"); - - // Create shared application state - let app_state = AppState { - s3_client, - user_id, - cluster_id, - }; - - // Build the Axum app - let app = Router::new() - .route("/health", get(health_handler)) - .route("/redeploy", post(redeploy_handler)) - .layer( - ServiceBuilder::new() - .layer(TraceLayer::new_for_http()) - ) - .with_state(app_state); - - // run server on specified port - let port = std::env::var("PORT").unwrap_or("6900".to_string()); - let addr: SocketAddr = format!("0.0.0.0:{}", port).parse().unwrap(); - - tracing::info!("Server listening on {}", addr); - - let listener = tokio::net::TcpListener::bind(&addr).await.map_err(|e| { - tracing::error!("Failed to bind to address {}: {}", addr, e); - AdminError::AdminConnectionError("Failed to bind to address".to_string(), e) - })?; - - axum::serve(listener, app).await.map_err(|e| { - tracing::error!("Server error: {}", e); - AdminError::AdminConnectionError("Server error".to_string(), - std::io::Error::new(std::io::ErrorKind::Other, e.to_string())) - })?; - - Ok(()) -} - -#[derive(Debug)] -pub enum AdminError { - AdminConnectionError(String, std::io::Error), - S3DownloadError( - String, - aws_sdk_s3::error::SdkError, - ), - CommandError(String, std::io::Error), - FileError(String, std::io::Error), - InvalidParameter(String), -} - -impl std::fmt::Display for AdminError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AdminError::AdminConnectionError(msg, err) => { - write!(f, "Connection error: {}: {}", msg, err) - } - AdminError::S3DownloadError(msg, err) => write!(f, "S3 error: {}: {}", msg, err), - AdminError::CommandError(msg, err) => write!(f, "Command error: {}: {}", msg, err), - AdminError::FileError(msg, err) => write!(f, "File error: {}: {}", msg, err), - AdminError::InvalidParameter(msg) => write!(f, "Invalid parameter: {}", msg), - } - } -} - -impl std::error::Error for AdminError {} diff --git a/helix-cli/Cargo.toml b/helix-cli/Cargo.toml index 1ce3bcf3..15d2f55f 100644 --- a/helix-cli/Cargo.toml +++ b/helix-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "helix-cli" -version = "1.0.115" +version = "1.0.118" edition = "2024" [dependencies] @@ -23,6 +23,7 @@ serde_json = "1.0" webbrowser = "1.0" tokio-tungstenite = "0.27.0" futures-util = "0.3.31" +anyhow = "1.0.98" [target.'cfg(windows)'.dependencies] windows = { version = "0.61.1", features = [ diff --git a/helix-cli/src/main.rs b/helix-cli/src/main.rs index e6d1c015..c6fc1060 100644 --- a/helix-cli/src/main.rs +++ b/helix-cli/src/main.rs @@ -9,7 +9,7 @@ use helix_db::{helix_engine::graph_core::config::Config, utils::styled_string::S use spinners::{Spinner, Spinners}; use std::{ fmt::Write, - fs::{self, OpenOptions, read_to_string}, + fs::{self, read_to_string, OpenOptions}, io::Write as iWrite, path::{Path, PathBuf}, process::Command, @@ -21,7 +21,7 @@ mod types; mod utils; #[tokio::main] -async fn main() { +async fn main() -> Result<(), ()> { check_helix_version().await; let args = HelixCli::parse(); @@ -31,9 +31,9 @@ async fn main() { Ok(_) => {} Err(_) => { println!("{}", "Cargo is not installed".red().bold()); - return; + return Err(()); } - } + }; match check_helix_installation() { Some(_) => {} @@ -44,7 +44,7 @@ async fn main() { .red() .bold() ); - return; + return Err(()); } }; @@ -53,7 +53,7 @@ async fn main() { && command.cluster.is_none() { println!("{}", "No path or instance specified!".red().bold()); - return; + return Err(()); } // -- helix start -- @@ -78,9 +78,10 @@ async fn main() { "Failed to start instance".red().bold() )); println!("└── {} {}", "Error:".red().bold(), e); + return Err(()); } } - return; + return Ok(()); } let output = dirs::home_dir() @@ -110,7 +111,7 @@ async fn main() { "No available ports found starting from".red().bold(), start_port ); - return; + return Err(()); } }; @@ -119,18 +120,18 @@ async fn main() { Ok(files) if !files.is_empty() => files, Ok(_) => { println!("{}", "No queries found, nothing to compile".red().bold()); - return; + return Err(()); } Err(e) => { println!("{} {}", "Error:".red().bold(), e); - return; + return Err(()); } }; if !command.remote { let code = match compile_and_build_helix(path, &output, files) { Ok(code) => code, - Err(_) => return, + Err(_) => return Err(()), }; if command.cluster.is_some() @@ -138,9 +139,9 @@ async fn main() { { match redeploy_helix(command.cluster.unwrap(), code) { Ok(_) => {} - Err(_) => return, + Err(_) => return Err(()), } - return; + return Err(()); } // -- helix deploy -- @@ -149,15 +150,15 @@ async fn main() { { match deploy_helix(port, code, None) { Ok(_) => {} - Err(_) => return, + Err(_) => return Err(()), } - return; + return Err(()); } } else { if let Some(cluster) = command.cluster { match redeploy_helix_remote(cluster, path, files).await { Ok(_) => {} - Err(_) => return, + Err(_) => return Err(()), } } else { println!( @@ -166,7 +167,7 @@ async fn main() { .red() .bold() ); - return; + return Err(()); } } } @@ -181,7 +182,7 @@ async fn main() { .red() .bold() ); - return; + return Err(()); } }; @@ -190,7 +191,7 @@ async fn main() { Some(dir) => dir, None => { println!("{}", "Could not determine home directory".red().bold()); - return; + return Err(()); } }; home_dir.join(".helix/repo/helix-db/helixdb") @@ -213,7 +214,7 @@ async fn main() { "Failed fetching the local cli version".red().bold(), e ); - return; + return Err(()); } }; let local_helix_version = match get_crate_version(&repo_path) { @@ -224,7 +225,7 @@ async fn main() { "Failed fetching the local db version".red().bold(), e ); - return; + return Err(()); } }; let remote_helix_version = get_remote_helix_version().await.unwrap(); @@ -253,7 +254,7 @@ async fn main() { .bold(), e ); - return; + return Err(()); } } @@ -272,7 +273,7 @@ async fn main() { "Error while pulling new helix-db version:".red().bold(), e ); - return; + return Err(()); } } @@ -289,7 +290,7 @@ async fn main() { "Error while installing new helix-cli version:".red().bold(), e ); - return; + return Err(()); } } } else { @@ -320,7 +321,7 @@ async fn main() { Err(e) => { sp.stop_with_message(format!("{}", "Failed to read files".red().bold())); println!("└── {}", e); - return; + return Err(()); } }; @@ -329,14 +330,14 @@ async fn main() { "{}", "No queries found, nothing to compile".red().bold() )); - return; + return Err(()); } - let analyzed_source = match generate(&files) { + let analyzed_source = match generate(&files, &path) { Ok((_, analyzed_source)) => analyzed_source, Err(e) => { sp.stop_with_message(format!("{}", e.to_string().red().bold())); - return; + return Err(()); } }; @@ -346,7 +347,7 @@ async fn main() { Err(e) => { println!("{} {}", "Failed to write typescript types".red().bold(), e); println!("└── {} {}", "Error:".red().bold(), e); - return; + return Err(()); } }; } @@ -361,7 +362,7 @@ async fn main() { Err(e) => { println!("{}", "Failed to transpile queries".red().bold()); println!("└── {} {}", "Error:".red().bold(), e); - return; + return Err(()); } } @@ -374,7 +375,7 @@ async fn main() { Err(e) => { println!("{} {}", "Failed to write queries file".red().bold(), e); println!("└── {} {}", "Error:".red().bold(), e); - return; + return Err(()); } } } @@ -398,7 +399,7 @@ async fn main() { Err(e) => { sp.stop_with_message(format!("{}", "Error checking files".red().bold())); println!("└── {}", e); - return; + return Err(()); } }; @@ -407,15 +408,15 @@ async fn main() { "{}", "No queries found, nothing to compile".red().bold() )); - return; + return Err(()); } - match generate(&files) { + match generate(&files, &path) { Ok(_) => {} Err(e) => { sp.stop_with_message(format!("{}", "Failed to generate queries".red().bold())); println!("└── {}", e); - return; + return Err(()); } } @@ -432,7 +433,7 @@ async fn main() { Ok(_) => {} Err(_) => { println!("{}", "Cargo is not installed".red().bold()); - return; + return Err(()); } } @@ -440,7 +441,7 @@ async fn main() { Ok(_) => {} Err(_) => { println!("{}", "Git is not installed".red().bold()); - return; + return Err(()); } } @@ -450,7 +451,7 @@ async fn main() { Some(dir) => dir, None => { println!("{}", "Could not determine home directory".red().bold()); - return; + return Err(()); } }; home_dir.join(".helix/repo") @@ -469,7 +470,7 @@ async fn main() { .yellow() .bold(), ); - return; + return Err(()); } match fs::create_dir_all(&repo_path) { @@ -482,7 +483,7 @@ async fn main() { println!("{}", "Failed to create directory structure".red().bold()); println!("|"); println!("└── {}", e); - return; + return Err(()); } } @@ -517,7 +518,7 @@ async fn main() { println!("{}", "Failed to install Helix repo".red().bold()); println!("|"); println!("└── {}", e); - return; + return Err(()); } } } @@ -537,7 +538,7 @@ async fn main() { "Queries already exist in".yellow().bold(), path_str ); - return; + return Err(()); } Ok(_) => {} Err(_) => {} @@ -567,7 +568,7 @@ async fn main() { Ok(instances) => { if instances.is_empty() { println!("{}", "No running Helix instances".yellow().bold()); - return; + return Err(()); } for instance in instances { print_instance(&instance); @@ -584,7 +585,7 @@ async fn main() { Ok(instances) => { if !instance_manager.running_instances().unwrap() { println!("{}", "No running Helix instances".bold()); - return; + return Err(()); } if command.all { println!("{}", "Stopping all running Helix instances".bold()); @@ -654,11 +655,11 @@ async fn main() { "No Helix instance found with id".red().bold(), iid.red().bold() ); - return; + return Err(()); } Err(e) => { println!("{} {}", "Error:".red().bold(), e); - return; + return Err(()); } } @@ -695,11 +696,11 @@ async fn main() { "No Helix instance found with id".red().bold(), iid.red().bold() ); - return; + return Err(()); } Err(e) => { println!("{} {}", "Error:".red().bold(), e); - return; + return Err(()); } } @@ -759,7 +760,7 @@ async fn main() { .red() .bold() ); - return; + return Err(()); } }; home_dir.join(".helix/repo/helix-db/helix-db") @@ -775,7 +776,7 @@ async fn main() { "Error while fetching the local cli version!".red().bold(), e ); - return; + return Err(()); } }; println!( @@ -806,7 +807,7 @@ async fn main() { "Failed to open graph visualizer for instance".red().bold(), iid.red().bold() ); - return; + return Err(()); } } Ok(None) => { @@ -815,11 +816,11 @@ async fn main() { "No Helix instance found with id".red().bold(), iid.red().bold() ); - return; + return Err(()); } Err(e) => { println!("{} {}", "Error:".red().bold(), e); - return; + return Err(()); } }; } @@ -878,4 +879,6 @@ async fn main() { } } } + + Ok(()) } diff --git a/helix-cli/src/utils.rs b/helix-cli/src/utils.rs index 1e569945..e96101dd 100644 --- a/helix-cli/src/utils.rs +++ b/helix-cli/src/utils.rs @@ -520,13 +520,21 @@ fn analyze_source(source: Source) -> Result { Ok(source) } -pub fn generate(files: &Vec) -> Result<(Content, GeneratedSource), String> { +pub fn generate(files: &Vec, path: &str) -> Result<(Content, GeneratedSource), String> { let mut content = generate_content(&files)?; content.source = parse_content(&content)?; - let analyzed_source = analyze_source(content.source.clone())?; + let mut analyzed_source = analyze_source(content.source.clone())?; + analyzed_source.config = read_config(&path)?; Ok((content, analyzed_source)) } +pub fn read_config(path: &str) -> Result { + let config_path = PathBuf::from(path).join("config.hx.json"); + let schema_path = PathBuf::from(path).join("schema.hx"); + let config = Config::from_files(config_path, schema_path).map_err(|e| e.to_string())?; + Ok(config) +} + pub fn gen_typescript(source: &GeneratedSource, output_path: &str) -> Result<(), String> { let mut file = match File::create(PathBuf::from(output_path).join("interface.d.ts")) { Ok(file) => file, @@ -614,7 +622,7 @@ pub fn compile_and_build_helix(path: String, output: &PathBuf, files: Vec (code, analyzer_source), Err(e) => { sp.stop_with_message(format!("{}", "Error compiling queries".red().bold())); diff --git a/helix-container/src/main.rs b/helix-container/src/main.rs index 8d005612..481c6c18 100644 --- a/helix-container/src/main.rs +++ b/helix-container/src/main.rs @@ -12,17 +12,9 @@ mod graphvis; mod queries; #[tokio::main] + async fn main() { - let home = dirs::home_dir().expect("Could not retrieve home directory"); - let config_path = home.join(".helix/repo/helix-db/helix-container/src/config.hx.json"); - let schema_path = home.join(".helix/repo/helix-db/helix-container/src/schema.hx"); - let config = match Config::from_files(config_path, schema_path) { - Ok(config) => config, - Err(e) => { - println!("Error loading config: {}", e); - Config::default() - } - }; + let config = queries::config().unwrap_or(Config::default()); let path = match std::env::var("HELIX_DATA_DIR") { Ok(val) => std::path::PathBuf::from(val).join("user"), diff --git a/helix-container/src/queries.rs b/helix-container/src/queries.rs index e69de29b..a38f25ca 100644 --- a/helix-container/src/queries.rs +++ b/helix-container/src/queries.rs @@ -0,0 +1,6 @@ +// DEFAULT CODE +use helix_db::helix_engine::graph_core::config::Config; + +pub fn config() -> Option { + None +} diff --git a/helix-db/Cargo.toml b/helix-db/Cargo.toml index 97516a6d..a72de1bb 100644 --- a/helix-db/Cargo.toml +++ b/helix-db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "helix-db" -version = "1.0.115" +version = "1.0.118" edition = "2024" description = "HelixDB is a powerful, open-source, graph-vector database built in Rust for intelligent data storage for RAG and AI." license = "AGPL-3.0" @@ -37,6 +37,7 @@ reqwest = { version = "0.12.15", features = [ "blocking", ], optional = true } url = { version = "2.5", optional = true } +tokio-util = { version = "0.7.15", features = ["compat"] } [dev-dependencies] rand = "0.9.0" diff --git a/helix-db/src/grammar.pest b/helix-db/src/grammar.pest index 9a6ac86e..738b38d4 100644 --- a/helix-db/src/grammar.pest +++ b/helix-db/src/grammar.pest @@ -29,14 +29,14 @@ properties = { "Properties" ~ ":" ~ "{" ~ field_defs? ~ "}" } query_def = { mcp_macro? ~ "QUERY" ~ identifier ~ query_params ~ "=>" ~ query_body ~ return_stmt } // TODO: possible optional return stmt query_params = { "(" ~ (param_def ~ ("," ~ param_def)*)? ~ ")" } param_def = { identifier ~ ":" ~ param_type } -query_body = { (get_stmt | AddN | AddV | BatchAddV | AddE | drop | for_loop)* } +query_body = { (get_stmt | drop | for_loop | creation_stmt)* } // --------------------------------------------------------------------- // Assignments and traversals // --------------------------------------------------------------------- get_stmt = { identifier ~ "<-" ~ evaluates_to_anything } -traversal = { (start_node | start_edge | start_vector ) ~ step* ~ last_step? } +traversal = { (start_node | start_edge ) ~ step* ~ last_step? } id_traversal = { identifier ~ ((step+ ~ last_step?) | last_step) } anonymous_traversal = { "_" ~ ((step+ ~ last_step?) | last_step)? } step = { "::" ~ (graph_step | order_by_asc | order_by_desc | where_step | closure_step | object_step | exclude_field | count | ID | range_step | AddE) } @@ -50,13 +50,20 @@ object_destructuring = { "{" ~ identifier ~ ("," ~ identifier)* ~ "}" } // --------------------------------------------------------------------- // Evaluation rules for different types // --------------------------------------------------------------------- +creation_stmt = { + AddN + | AddV + | BatchAddV + | AddE +} + evaluates_to_anything = { - AddN + AddN | AddV | BatchAddV + | AddE | search_vector | bm25_search - | AddE | exists | none | traversal @@ -159,7 +166,7 @@ none = { "NONE" } ID = { "ID" } update_field = { identifier ~ ":" ~ (evaluates_to_anything | anonymous_traversal) } update = { "UPDATE" ~ "(" ~ "{" ~ update_field ~ ("," ~ update_field)* ~ "}" ~ ")" } -drop = { "DROP" ~ (traversal | id_traversal | identifier)? } +drop = { "DROP" ~ evaluates_to_anything } // --------------------------------------------------------------------- // Vector steps @@ -187,8 +194,7 @@ NEQ = { "NEQ" ~ "(" ~ (evaluates_to_anything | anonymous_traversal) // --------------------------------------------------------------------- // Object access and remapping steps // --------------------------------------------------------------------- -object_step = { "{" ~ ((mapping_field ~ ("," ~ mapping_field)* ~ ","?) | -(mapping_field ~ ("," ~ mapping_field)* ~ ","?)? ~ spread_object) ~ "}" } +object_step = { "{" ~ ((mapping_field ~ ("," ~ mapping_field)* ~ "," ~ spread_object ~ ","?) | (mapping_field ~ ("," ~ mapping_field)* ~ ","?) ) ~ "}" } exclude_field = { "!" ~ "{" ~ identifier ~ ("," ~ identifier)* ~ ("," ~ spread_object)? ~ "}" } closure_step = { "|" ~ identifier ~ "|" ~ object_step } spread_object = { ".." ~ ","?} diff --git a/helix-db/src/helix_engine/bm25/bm25.rs b/helix-db/src/helix_engine/bm25/bm25.rs index de70092f..99f12a30 100644 --- a/helix-db/src/helix_engine/bm25/bm25.rs +++ b/helix-db/src/helix_engine/bm25/bm25.rs @@ -344,13 +344,13 @@ impl BM25 for HBM25Config { pub trait HybridSearch { /// Search both hnsw index and bm25 docs - async fn hybrid_search( + fn hybrid_search( self, query: &str, query_vector: &[f64], alpha: f32, limit: usize, - ) -> Result, GraphError>; + ) -> impl std::future::Future, GraphError>> + Send; } impl HybridSearch for HelixGraphStorage { diff --git a/helix-db/src/helix_engine/graph_core/config.rs b/helix-db/src/helix_engine/graph_core/config.rs index 6614989d..e9ee05ca 100644 --- a/helix-db/src/helix_engine/graph_core/config.rs +++ b/helix-db/src/helix_engine/graph_core/config.rs @@ -5,25 +5,43 @@ use std::{ }; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct VectorConfig { pub m: Option, pub ef_construction: Option, pub ef_search: Option, } -#[derive(Serialize, Deserialize, Debug)] +impl Default for VectorConfig { + fn default() -> Self { + Self { + m: Some(16), + ef_construction: Some(128), + ef_search: Some(768), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct GraphConfig { pub secondary_indices: Option>, } +impl Default for GraphConfig { + fn default() -> Self { + Self { + secondary_indices: None, + } + } +} + #[derive(Serialize, Deserialize, Debug)] pub struct Config { - pub vector_config: VectorConfig, - pub graph_config: GraphConfig, + pub vector_config: Option, + pub graph_config: Option, pub db_max_size_gb: Option, - pub mcp: bool, - pub bm25: bool, + pub mcp: Option, + pub bm25: Option, pub schema: Option, pub embedding_model: Option, pub graphvis_node_label: Option, @@ -42,17 +60,17 @@ impl Config { graphvis_node_label: Option, ) -> Self { Self { - vector_config: VectorConfig { + vector_config: Some(VectorConfig { m: Some(m), ef_construction: Some(ef_construction), ef_search: Some(ef_search), - }, - graph_config: GraphConfig { + }), + graph_config: Some(GraphConfig { secondary_indices: None, - }, + }), db_max_size_gb: Some(db_max_size_gb), - mcp, - bm25, + mcp: Some(mcp), + bm25: Some(bm25), schema, embedding_model, graphvis_node_label, @@ -102,22 +120,46 @@ impl Config { pub fn to_json(&self) -> String { sonic_rs::to_string_pretty(self).unwrap() } + + pub fn get_vector_config(&self) -> VectorConfig { + self.vector_config.clone().unwrap_or(VectorConfig::default()) + } + + pub fn get_graph_config(&self) -> GraphConfig { + self.graph_config.clone().unwrap_or(GraphConfig::default()) + } + + pub fn get_db_max_size_gb(&self) -> usize { + self.db_max_size_gb.unwrap_or(10) + } + + pub fn get_mcp(&self) -> bool { + self.mcp.unwrap_or(true) + } + + pub fn get_bm25(&self) -> bool { + self.bm25.unwrap_or(true) + } + + pub fn get_schema(&self) -> Option { + self.schema.clone() + } } impl Default for Config { fn default() -> Self { Self { - vector_config: VectorConfig { + vector_config: Some(VectorConfig { m: Some(16), ef_construction: Some(128), ef_search: Some(768), - }, - graph_config: GraphConfig { + }), + graph_config: Some(GraphConfig { secondary_indices: None, - }, + }), db_max_size_gb: Some(10), - mcp: true, - bm25: true, + mcp: Some(true), + bm25: Some(true), schema: None, embedding_model: Some("text-embedding-ada-002".to_string()), graphvis_node_label: None, @@ -129,25 +171,35 @@ impl fmt::Display for Config { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, - "Vector config => m: {:?}, ef_construction: {:?}, ef_search: {:?}\n - Graph config => secondary_indicies: {:?}\n - db_max_size_gb: {:?}\n - mcp: {:?}\n - bm25: {:?}\n - schema: {:?}\n - embedding_model: {:?}\n - graphvis_node_label: {:?}", - self.vector_config.m, - self.vector_config.ef_construction, - self.vector_config.ef_search, - self.graph_config.secondary_indices, - self.db_max_size_gb, - self.mcp, - self.bm25, - self.schema, - self.embedding_model, - self.graphvis_node_label, - ) + "pub fn config() -> Option {{" + )?; + write!(f, "return Some(Config {{")?; + write!(f, "vector_config: Some(VectorConfig {{")?; + write!(f, "m: Some({}),", self.vector_config.as_ref().unwrap().m.unwrap_or(16))?; + write!(f, "ef_construction: Some({}),", self.vector_config.as_ref().unwrap().ef_construction.unwrap_or(128))?; + write!(f, "ef_search: Some({}),", self.vector_config.as_ref().unwrap().ef_search.unwrap_or(768))?; + write!(f, "}}),")?; + write!(f, "graph_config: Some(GraphConfig {{")?; + write!(f, "secondary_indices: {},", match &self.graph_config.as_ref().unwrap().secondary_indices { + Some(indices) => format!("Some(vec![{}])", indices.iter().map(|s| format!("\"{}\"", s)).collect::>().join(", ")), + None => "None".to_string(), + })?; + write!(f, "}}),")?; + write!(f, "db_max_size_gb: Some({}),", self.db_max_size_gb.unwrap_or(10))?; + write!(f, "mcp: Some({}),", self.mcp.unwrap_or(true))?; + write!(f, "bm25: Some({}),", self.bm25.unwrap_or(true))?; + write!(f, "schema: None,")?; + write!(f, "embedding_model: {},", match &self.embedding_model { + Some(model) => format!("Some(\"{}\".to_string())", model), + None => "None".to_string(), + })?; + write!(f, "graphvis_node_label: {},", match &self.graphvis_node_label { + Some(label) => format!("Some(\"{}\".to_string())", label), + None => "None".to_string(), + })?; + write!(f, "}})")?; + write!(f, "}}")?; + Ok(()) } } diff --git a/helix-db/src/helix_engine/graph_core/graph_core.rs b/helix-db/src/helix_engine/graph_core/graph_core.rs index cf3cdec3..aa1ca1dc 100644 --- a/helix-db/src/helix_engine/graph_core/graph_core.rs +++ b/helix-db/src/helix_engine/graph_core/graph_core.rs @@ -40,7 +40,7 @@ impl HelixGraphEngine { Err(err) => return Err(err), }; - let (mcp_backend, mcp_connections) = if should_use_mcp { + let (mcp_backend, mcp_connections) = if should_use_mcp.unwrap_or(false) { let mcp_backend = Arc::new(McpBackend::new(storage.clone())); let mcp_connections = Arc::new(Mutex::new(McpConnections::new())); (Some(mcp_backend), Some(mcp_connections)) diff --git a/helix-db/src/helix_engine/graph_core/ops/tr_val.rs b/helix-db/src/helix_engine/graph_core/ops/tr_val.rs index d8ba8d31..954ca050 100644 --- a/helix-db/src/helix_engine/graph_core/ops/tr_val.rs +++ b/helix-db/src/helix_engine/graph_core/ops/tr_val.rs @@ -7,7 +7,7 @@ use crate::{ items::{Edge, Node}, }, }; -use std::hash::Hash; +use std::{borrow::Cow, hash::Hash}; #[derive(Clone, Debug)] pub enum TraversalVal { @@ -68,7 +68,7 @@ impl IntoIterator for TraversalVal { pub trait Traversable { fn id(&self) -> u128; fn label(&self) -> String; - fn check_property(&self, prop: &str) -> Result<&Value, GraphError>; + fn check_property(&self, prop: &str) -> Result, GraphError>; fn uuid(&self) -> String; } @@ -105,7 +105,7 @@ impl Traversable for TraversalVal { } } - fn check_property(&self, prop: &str) -> Result<&Value, GraphError> { + fn check_property(&self, prop: &str) -> Result, GraphError> { match self { TraversalVal::Node(node) => node.check_property(prop), TraversalVal::Edge(edge) => edge.check_property(prop), @@ -132,7 +132,7 @@ impl Traversable for Vec { self[0].label() } - fn check_property(&self, prop: &str) -> Result<&Value, GraphError> { + fn check_property(&self, prop: &str) -> Result, GraphError> { if self.is_empty() { return Err(GraphError::ConversionError(format!( "Invalid traversal value" diff --git a/helix-db/src/helix_engine/graph_core/ops/util/drop.rs b/helix-db/src/helix_engine/graph_core/ops/util/drop.rs index 04edbb8a..f9629ac0 100644 --- a/helix-db/src/helix_engine/graph_core/ops/util/drop.rs +++ b/helix-db/src/helix_engine/graph_core/ops/util/drop.rs @@ -1,5 +1,9 @@ use crate::helix_engine::{ - bm25::bm25::BM25, graph_core::ops::tr_val::TraversalVal, storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError + bm25::bm25::BM25, + graph_core::ops::tr_val::TraversalVal, + storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, + types::GraphError, + vector_core::hnsw::HNSW, }; use heed3::RwTxn; use std::{fmt::Debug, sync::Arc}; @@ -17,7 +21,6 @@ where storage: Arc, txn: &mut RwTxn, ) -> Result<(), GraphError> { - println!("Dropping traversal {:?}", iter); iter.into_iter() .try_for_each(|item| -> Result<(), GraphError> { match item { @@ -36,7 +39,10 @@ where Ok(_) => Ok(()), Err(e) => return Err(e), }, - TraversalVal::Vector(_) => Ok(()), + TraversalVal::Vector(vector) => match storage.vectors.delete(txn, vector.id) { + Ok(_) => Ok(()), + Err(e) => return Err(e.into()), + }, _ => { return Err(GraphError::ConversionError(format!( "Incorrect Type: {:?}", @@ -47,4 +53,3 @@ where }) } } - diff --git a/helix-db/src/helix_engine/graph_core/ops/util/props.rs b/helix-db/src/helix_engine/graph_core/ops/util/props.rs index 22b4284b..a8cdd30a 100644 --- a/helix-db/src/helix_engine/graph_core/ops/util/props.rs +++ b/helix-db/src/helix_engine/graph_core/ops/util/props.rs @@ -1,10 +1,10 @@ -use crate::helix_engine::{ +use crate::{helix_engine::{ graph_core::{ ops::tr_val::TraversalVal, traversal_iter::{RoTraversalIterator, RwTraversalIterator}, }, types::GraphError, -}; +}, utils::filterable::Filterable}; pub struct PropsIterator<'a, I> { iter: I, @@ -20,36 +20,17 @@ where fn next(&mut self) -> Option { match self.iter.next() { - Some(Ok(TraversalVal::Node(node))) => match node.properties { - Some(prop) => { - let prop = prop.get(self.prop); - match prop { - Some(prop) => Some(Ok(TraversalVal::Value(prop.clone()))), - None => None, - } - } - None => None, + Some(Ok(TraversalVal::Node(node))) => match node.check_property(self.prop) { + Ok(prop) => Some(Ok(TraversalVal::Value(prop.into_owned()))), + Err(e) => Some(Err(e)), }, - Some(Ok(TraversalVal::Edge(edge))) => match edge.properties { - Some(prop) => { - let prop = prop.get(self.prop); - println!("prop: {:?}", prop); - match prop { - Some(prop) => Some(Ok(TraversalVal::Value(prop.clone()))), - None => None, - } - } - None => None, + Some(Ok(TraversalVal::Edge(edge))) => match edge.check_property(self.prop) { + Ok(prop) => Some(Ok(TraversalVal::Value(prop.into_owned()))), + Err(e) => Some(Err(e)), }, - Some(Ok(TraversalVal::Vector(vec))) => match vec.properties { - Some(prop) => { - let prop = prop.get(self.prop); - match prop { - Some(prop) => Some(Ok(TraversalVal::Value(prop.clone()))), - None => None, - } - } - None => None, + Some(Ok(TraversalVal::Vector(vec))) => match vec.check_property(self.prop) { + Ok(prop) => Some(Ok(TraversalVal::Value(prop.into_owned()))), + Err(e) => Some(Err(e)), }, _ => None, } diff --git a/helix-db/src/helix_engine/graph_core/ops/util/range.rs b/helix-db/src/helix_engine/graph_core/ops/util/range.rs index a776f81a..f880ff6a 100644 --- a/helix-db/src/helix_engine/graph_core/ops/util/range.rs +++ b/helix-db/src/helix_engine/graph_core/ops/util/range.rs @@ -57,36 +57,47 @@ pub trait RangeAdapter<'a>: Iterator { /// ```rust /// let traversal = G::new(storage, &txn).range(0, 10); /// ``` - fn range( + fn range( self, - start: usize, - end: usize, + start: N, + end: K, ) -> RoTraversalIterator<'a, impl Iterator>> where Self: Sized + Iterator, - Self::Item: Send; + Self::Item: Send, + N: TryInto, + K: TryInto, + N::Error: std::fmt::Debug, + K::Error: std::fmt::Debug; } impl<'a, I: Iterator> + 'a> RangeAdapter<'a> for RoTraversalIterator<'a, I> { #[inline(always)] - fn range( + fn range( self, - start: usize, - end: usize, + start: N, + end: K, ) -> RoTraversalIterator<'a, impl Iterator>> where Self: Sized + Iterator, Self::Item: Send, + N: TryInto, + K: TryInto, + N::Error: std::fmt::Debug, + K::Error: std::fmt::Debug, { { + let start_usize = start.try_into().expect("Start index must be non-negative and fit in usize"); + let end_usize = end.try_into().expect("End index must be non-negative and fit in usize"); + RoTraversalIterator { inner: Range { iter: self.inner, curr_idx: 0, - start, - end, + start: start_usize, + end: end_usize, }, storage: Arc::clone(&self.storage), txn: self.txn, diff --git a/helix-db/src/helix_engine/graph_core/ops/vectors/insert.rs b/helix-db/src/helix_engine/graph_core/ops/vectors/insert.rs index 23a2b044..93b56b57 100644 --- a/helix-db/src/helix_engine/graph_core/ops/vectors/insert.rs +++ b/helix-db/src/helix_engine/graph_core/ops/vectors/insert.rs @@ -57,12 +57,13 @@ impl<'a, 'b, I: Iterator>> InsertVAdapte let fields = match fields { Some(mut fields) => { fields.push((String::from("label"), Value::String(label.to_string()))); + fields.push((String::from("is_deleted"), Value::Boolean(false))); Some(fields) } - None => Some(vec![( - String::from("label"), - Value::String(label.to_string()), - )]), + None => Some(vec![ + (String::from("label"), Value::String(label.to_string())), + (String::from("is_deleted"), Value::Boolean(false)), + ]), }; let vector = self.storage.vectors.insert::(self.txn, &query, fields); @@ -106,4 +107,3 @@ impl<'a, 'b, I: Iterator>> InsertVAdapte } } } - diff --git a/helix-db/src/helix_engine/graph_core/ops/vectors/search.rs b/helix-db/src/helix_engine/graph_core/ops/vectors/search.rs index 773cdf70..0383df63 100644 --- a/helix-db/src/helix_engine/graph_core/ops/vectors/search.rs +++ b/helix-db/src/helix_engine/graph_core/ops/vectors/search.rs @@ -1,12 +1,10 @@ use heed3::RoTxn; use super::super::tr_val::TraversalVal; -use crate::{ - helix_engine::{ - graph_core::traversal_iter::RoTraversalIterator, - types::{GraphError, VectorError}, - vector_core::{hnsw::HNSW, vector::HVector}, - }, +use crate::helix_engine::{ + graph_core::traversal_iter::RoTraversalIterator, + types::{GraphError, VectorError}, + vector_core::{hnsw::HNSW, vector::HVector}, }; use helix_macros::debug_trace; use std::iter::once; @@ -84,6 +82,11 @@ impl<'a, I: Iterator> + 'a> SearchVAdapt let error = GraphError::VectorError("invalid vector dimensions!".to_string()); once(Err(error)).collect::>().into_iter() } + Err(VectorError::VectorAlreadyDeleted(id)) => { + let error = + GraphError::VectorError(format!("vector already deleted for id {}", id)); + once(Err(error)).collect::>().into_iter() + } .collect::>() .into_iter(), }; @@ -97,4 +100,3 @@ impl<'a, I: Iterator> + 'a> SearchVAdapt } } } - diff --git a/helix-db/src/helix_engine/graph_core/traversal_iter.rs b/helix-db/src/helix_engine/graph_core/traversal_iter.rs index 171a0189..6cd92b19 100644 --- a/helix-db/src/helix_engine/graph_core/traversal_iter.rs +++ b/helix-db/src/helix_engine/graph_core/traversal_iter.rs @@ -67,7 +67,7 @@ impl<'a, I: Iterator>> RoTraversalIterat .sorted_by(|a, b| match (a, b) { (TraversalVal::Node(a), TraversalVal::Node(b)) => { match (a.check_property(property), b.check_property(property)) { - (Ok(val_a), Ok(val_b)) => val_a.cmp(val_b), + (Ok(val_a), Ok(val_b)) => val_a.cmp(&val_b), (Ok(_), Err(_)) => Ordering::Less, (Err(_), Ok(_)) => Ordering::Greater, (Err(_), Err(_)) => Ordering::Equal, @@ -75,7 +75,7 @@ impl<'a, I: Iterator>> RoTraversalIterat } (TraversalVal::Edge(a), TraversalVal::Edge(b)) => { match (a.check_property(property), b.check_property(property)) { - (Ok(val_a), Ok(val_b)) => val_a.cmp(val_b), + (Ok(val_a), Ok(val_b)) => val_a.cmp(&val_b), (Ok(_), Err(_)) => Ordering::Less, (Err(_), Ok(_)) => Ordering::Greater, (Err(_), Err(_)) => Ordering::Equal, @@ -83,7 +83,7 @@ impl<'a, I: Iterator>> RoTraversalIterat } (TraversalVal::Vector(a), TraversalVal::Vector(b)) => { match (a.check_property(property), b.check_property(property)) { - (Ok(val_a), Ok(val_b)) => val_a.cmp(val_b), + (Ok(val_a), Ok(val_b)) => val_a.cmp(&val_b), (Ok(_), Err(_)) => Ordering::Less, (Err(_), Ok(_)) => Ordering::Greater, (Err(_), Err(_)) => Ordering::Equal, @@ -105,7 +105,7 @@ impl<'a, I: Iterator>> RoTraversalIterat .sorted_by(|a, b| match (a, b) { (TraversalVal::Node(a), TraversalVal::Node(b)) => { match (a.check_property(property), b.check_property(property)) { - (Ok(val_a), Ok(val_b)) => val_b.cmp(val_a), + (Ok(val_a), Ok(val_b)) => val_b.cmp(&val_a), (Ok(_), Err(_)) => Ordering::Greater, (Err(_), Ok(_)) => Ordering::Less, (Err(_), Err(_)) => Ordering::Equal, @@ -113,7 +113,7 @@ impl<'a, I: Iterator>> RoTraversalIterat } (TraversalVal::Edge(a), TraversalVal::Edge(b)) => { match (a.check_property(property), b.check_property(property)) { - (Ok(val_a), Ok(val_b)) => val_b.cmp(val_a), + (Ok(val_a), Ok(val_b)) => val_b.cmp(&val_a), (Ok(_), Err(_)) => Ordering::Greater, (Err(_), Ok(_)) => Ordering::Less, (Err(_), Err(_)) => Ordering::Equal, @@ -121,7 +121,7 @@ impl<'a, I: Iterator>> RoTraversalIterat } (TraversalVal::Vector(a), TraversalVal::Vector(b)) => { match (a.check_property(property), b.check_property(property)) { - (Ok(val_a), Ok(val_b)) => val_b.cmp(val_a), + (Ok(val_a), Ok(val_b)) => val_b.cmp(&val_a), (Ok(_), Err(_)) => Ordering::Greater, (Err(_), Ok(_)) => Ordering::Less, (Err(_), Err(_)) => Ordering::Equal, diff --git a/helix-db/src/helix_engine/graph_core/traversal_tests.rs b/helix-db/src/helix_engine/graph_core/traversal_tests.rs index 36e259ab..44612e83 100644 --- a/helix-db/src/helix_engine/graph_core/traversal_tests.rs +++ b/helix-db/src/helix_engine/graph_core/traversal_tests.rs @@ -1,24 +1,33 @@ -use std::{sync::Arc, time::Instant}; - -use crate::helix_engine::{ - graph_core::ops::{ - g::G, - in_::{in_e::InEdgesAdapter, to_n::ToNAdapter, to_v::ToVAdapter}, - out::{from_n::FromNAdapter, from_v::FromVAdapter, out::OutAdapter}, - source::{add_n::AddNAdapter, e_from_id::EFromIdAdapter, n_from_id::NFromIdAdapter}, - tr_val::{Traversable, TraversalVal}, - util::{dedup::DedupAdapter, props::PropsAdapter, range::RangeAdapter}, - vectors::brute_force_search::BruteForceSearchVAdapter, +use std::{collections::HashMap, str::FromStr, sync::Arc, time::Instant}; + +use crate::{ + exclude_field, + helix_engine::{ + graph_core::ops::{ + g::G, + in_::{in_e::InEdgesAdapter, to_n::ToNAdapter, to_v::ToVAdapter}, + out::{from_n::FromNAdapter, from_v::FromVAdapter, out::OutAdapter}, + source::{add_n::AddNAdapter, e_from_id::EFromIdAdapter, n_from_id::NFromIdAdapter}, + tr_val::{Traversable, TraversalVal}, + util::{ + dedup::DedupAdapter, map::MapAdapter, props::PropsAdapter, range::RangeAdapter, + }, + vectors::brute_force_search::BruteForceSearchVAdapter, + }, + storage_core::storage_core::HelixGraphStorage, + types::GraphError, vector_core::vector, + }, + protocol::{ + remapping::{Remapping, RemappingMap, ResponseRemapping}, + return_values::ReturnValue, }, - storage_core::storage_core::HelixGraphStorage, - types::GraphError, }; use crate::{ helix_engine::graph_core::ops::{ source::n_from_type::NFromTypeAdapter, util::paths::ShortestPathAdapter, }, - utils::{filterable::Filterable, id::ID}, protocol::value::Value, + utils::{filterable::Filterable, id::ID}, }; use crate::{ helix_engine::{ @@ -34,6 +43,7 @@ use crate::{ use heed3::RoTxn; use rand::Rng; use serde::{Deserialize, Serialize}; +use sonic_rs::JsonValueTrait; use tempfile::TempDir; use super::ops::{ @@ -836,7 +846,7 @@ fn test_filter_nodes() { .filter_ref(|val, _| { if let Ok(TraversalVal::Node(node)) = val { if let Ok(value) = node.check_property("age") { - match value { + match value.as_ref() { Value::F64(age) => Ok(*age > 30.0), Value::I32(age) => Ok(*age > 30), _ => Ok(false), @@ -885,8 +895,8 @@ fn test_filter_macro_single_argument() { .iter() .any(|val| if let TraversalVal::Node(node) = val { let name = node.check_property("name").unwrap(); - name == &Value::String("Alice".to_string()) - || name == &Value::String("Bob".to_string()) + name.as_ref() == &Value::String("Alice".to_string()) + || name.as_ref() == &Value::String("Bob".to_string()) } else { false }) @@ -912,7 +922,7 @@ fn test_filter_macro_multiple_arguments() { ) -> Result { if let Ok(TraversalVal::Node(node)) = val { if let Ok(value) = node.check_property("age") { - match value { + match value.as_ref() { Value::F64(age) => Ok(*age > min_age as f64), Value::I32(age) => Ok(*age > min_age), _ => Ok(false), @@ -974,7 +984,7 @@ fn test_filter_edges() { fn recent_edge(val: &Result, year: i32) -> Result { if let Ok(TraversalVal::Edge(edge)) = val { if let Ok(value) = edge.check_property("since") { - match value { + match value.as_ref() { Value::I32(since) => return Ok(*since > year), Value::F64(since) => return Ok(*since > year as f64), _ => return Ok(false), @@ -1012,7 +1022,7 @@ fn test_filter_empty_result() { .filter_ref(|val, _| { if let Ok(TraversalVal::Node(node)) = val { if let Ok(value) = node.check_property("age") { - match value { + match value.as_ref() { Value::I32(age) => return Ok(*age > 100), Value::F64(age) => return Ok(*age > 100.0), _ => return Ok(false), @@ -1068,7 +1078,7 @@ fn test_filter_chain() { ) -> Result { if let Ok(TraversalVal::Node(node)) = val { if let Ok(value) = node.check_property("age") { - match value { + match value.as_ref() { Value::F64(age) => return Ok(*age > min_age as f64), Value::I32(age) => return Ok(*age > min_age), _ => return Ok(false), @@ -1333,7 +1343,7 @@ fn test_update_node() { .collect_to::>(); assert_eq!(updated_users.len(), 1); assert_eq!( - updated_users[0].check_property("name").unwrap().to_string(), + updated_users[0].check_property("name").unwrap().into_owned().to_string(), "john" ); } @@ -1675,13 +1685,16 @@ fn test_add_e_with_dup_flag() { }; println!("random_nodes: {:?}", random_nodes.len()); - let now = Instant::now(); let mut txn = storage.graph_env.write_txn().unwrap(); for chunk in random_nodes.chunks(100000) { for (random_node1, random_node2) in chunk { if random_node1.id() == random_node2.id() { - println!("random_node1: {:?}, random_node2: {:?}", random_node1.id(), random_node2.id()); + println!( + "random_node1: {:?}, random_node2: {:?}", + random_node1.id(), + random_node2.id() + ); continue; } let _ = G::new_mut(Arc::clone(&storage), &mut txn) @@ -1700,9 +1713,6 @@ fn test_add_e_with_dup_flag() { let end = now.elapsed(); println!("10 mill took {:?}", end); - - - let start = Instant::now(); let txn = storage.graph_env.read_txn().unwrap(); let edge_length = storage.edges_db.len(&txn).unwrap(); @@ -1724,7 +1734,7 @@ fn test_add_e_with_dup_flag() { .filter_ref(|val, _| { if let Ok(TraversalVal::Node(node)) = val { node.check_property("age").map(|value| { - if let Value::I32(age) = value { + if let Value::I32(age) = value.as_ref() { *age < 10000 } else { false @@ -1740,8 +1750,6 @@ fn test_add_e_with_dup_flag() { println!("traversal: {:?}", traversal.len()); assert_eq!(count, 10000); - - } #[ignore] @@ -2206,3 +2214,101 @@ fn test_drop_traversal() { assert_eq!(traversal.len(), 0); } + +#[test] +fn test_exclude_field_remapping() { + let (storage, _temp_dir) = setup_test_db(); + let mut txn = storage.graph_env.write_txn().unwrap(); + + let node = G::new_mut(Arc::clone(&storage), &mut txn) + .add_n("person", Some(props! { "text" => "test", "other" => "other" }), None) + .collect_to_val(); + + let traversal = G::new(Arc::clone(&storage), &txn) + .n_from_type("person") + .collect_to::>(); + + let mut remapping_vals = RemappingMap::new(); + + let mut return_vals: HashMap = HashMap::new(); + return_vals.insert( + "files".to_string(), + ReturnValue::from_traversal_value_array_with_mixin( + G::new_from(Arc::clone(&storage), &txn, traversal.clone()) + .map_traversal(|item, txn| { + println!("item: {:?}", item); + exclude_field!(remapping_vals, item.clone(), "text")?; + Ok(item) + }) + .collect_to::>() + .clone(), + remapping_vals.borrow_mut(), + ), + ); + + assert_eq!(return_vals.len(), 1); + + + #[derive(Serialize, Deserialize)] + struct Test { + text: Option, + other: Option, + } + #[derive(Serialize, Deserialize)] + struct Response { + files: Vec, + } + let value = sonic_rs::to_vec(&return_vals).unwrap(); + let value: Response = sonic_rs::from_slice(&value).unwrap(); + let value = value.files.first().unwrap(); + + let expected = Test { + text: None, + other: Some("other".to_string()), + }; + + + assert_eq!(value.text, expected.text); + assert_eq!(value.other, expected.other); +} + +#[test] +fn test_delete_vector() { + let (storage, _temp_dir) = setup_test_db(); + let mut txn = storage.graph_env.write_txn().unwrap(); + + let vector = G::new_mut(Arc::clone(&storage), &mut txn) + .insert_v:: bool>(&vec![1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "vector", None) + .collect_to_val(); + + txn.commit().unwrap(); + + let txn = storage.graph_env.read_txn().unwrap(); + let traversal = G::new(Arc::clone(&storage), &txn) + .search_v:: bool>(&vec![1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 2000, None) + .collect_to::>(); + + txn.commit().unwrap(); + assert_eq!(traversal.len(), 1); + assert_eq!(traversal[0].id(), vector.id()); + + let mut txn = storage.graph_env.write_txn().unwrap(); + + Drop::drop_traversal( + G::new(Arc::clone(&storage), &txn) + .search_v:: bool>(&vec![1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 2000, None) + .collect_to::>(), + Arc::clone(&storage), + &mut txn, + ) + .unwrap(); + + txn.commit().unwrap(); + + let txn = storage.graph_env.read_txn().unwrap(); + let traversal = G::new(Arc::clone(&storage), &txn) + .search_v:: bool>(&vec![1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 2000, None) + .collect_to::>(); + + assert_eq!(traversal.len(), 0); +} \ No newline at end of file diff --git a/helix-db/src/helix_engine/macros.rs b/helix-db/src/helix_engine/macros.rs index 44c2ba5c..30d52f2f 100644 --- a/helix-db/src/helix_engine/macros.rs +++ b/helix-db/src/helix_engine/macros.rs @@ -131,7 +131,7 @@ pub mod macros { #[macro_export] macro_rules! field_remapping { - ($remapping_vals:expr, $var_name:expr, $old_name:expr => $new_name:expr) => {{ + ($remapping_vals:expr, $var_name:expr, $should_spread:expr, $old_name:expr => $new_name:expr) => {{ let old_value = match $var_name.check_property($old_name) { Ok(val) => val, Err(e) => { @@ -147,7 +147,7 @@ pub mod macros { $var_name.id(), ResponseRemapping::new( HashMap::from([($old_name.to_string(), old_value_remapping)]), - false, + $should_spread, ), ); Ok::($var_name) // Return the Ok value @@ -156,7 +156,7 @@ pub mod macros { #[macro_export] macro_rules! traversal_remapping { - ($remapping_vals:expr, $var_name:expr, $new_name:expr => $traversal:expr) => {{ + ($remapping_vals:expr, $var_name:expr, $should_spread:expr, $new_name:expr => $traversal:expr) => {{ // TODO: ref? let new_remapping = Remapping::new( false, @@ -167,7 +167,7 @@ pub mod macros { $var_name.id(), ResponseRemapping::new( HashMap::from([($new_name.to_string(), new_remapping)]), - false, + $should_spread, ), ); Ok::($var_name) @@ -181,16 +181,17 @@ pub mod macros { $( let field_to_exclude_remapping = Remapping::new( true, - Some($field_to_exclude.to_string()), + None, None, ); $remapping_vals.insert( $var_name.id(), ResponseRemapping::new( HashMap::from([($field_to_exclude.to_string(), field_to_exclude_remapping)]), - false, + true, ), ); + println!("inserting remapping: {:?}", $remapping_vals.borrow_mut()); )* Ok::($var_name) }}; @@ -198,7 +199,7 @@ pub mod macros { #[macro_export] macro_rules! identifier_remapping { - ($remapping_vals:expr, $var_name:expr, $field_name:expr => $identifier_value:expr) => {{ + ($remapping_vals:expr, $var_name:expr, $should_spread:expr, $field_name:expr => $identifier_value:expr) => {{ let value = match $var_name.check_property($field_name) { Ok(val) => val.clone(), // TODO: try and remove clone Err(e) => { @@ -212,12 +213,12 @@ pub mod macros { false, Some($identifier_value.to_string()), Some(ReturnValue::from(value)), - ); + );- $remapping_vals.insert( $var_name.id(), ResponseRemapping::new( HashMap::from([($field_name.to_string(), value_remapping)]), - false, + $should_spread, ), ); Ok::($var_name) @@ -226,7 +227,7 @@ pub mod macros { #[macro_export] macro_rules! value_remapping { - ($remapping_vals:expr, $var_name:expr, $field_name:expr => $value:expr) => {{ + ($remapping_vals:expr, $var_name:expr, $should_spread:expr, $field_name:expr => $value:expr) => {{ let value = match $var_name.check_property($field_name) { Ok(val) => val.clone(), Err(e) => { @@ -245,7 +246,7 @@ pub mod macros { $var_name.id(), ResponseRemapping::new( HashMap::from([($field_name.to_string(), old_value_remapping)]), - false, + $should_spread, ), ); Ok::($var_name) // Return the Ok value diff --git a/helix-db/src/helix_engine/storage_core/storage_core.rs b/helix-db/src/helix_engine/storage_core/storage_core.rs index f6fd0c49..4690de3d 100644 --- a/helix-db/src/helix_engine/storage_core/storage_core.rs +++ b/helix-db/src/helix_engine/storage_core/storage_core.rs @@ -2,7 +2,7 @@ use super::storage_methods::DBMethods; use crate::{ helix_engine::{ bm25::bm25::HBM25Config, - graph_core::config::Config, + graph_core::config::{Config, GraphConfig}, storage_core::storage_methods::StorageMethods, types::GraphError, vector_core::{ @@ -16,7 +16,7 @@ use crate::{ label_hash::hash_label, }, }; -use heed3::{byteorder::BE, types::*, Database, DatabaseFlags, Env, EnvOpenOptions, RoTxn, RwTxn}; +use heed3::{Database, DatabaseFlags, Env, EnvOpenOptions, RoTxn, RwTxn, byteorder::BE, types::*}; use std::{collections::HashMap, fs, path::Path}; // database names for different stores @@ -113,7 +113,7 @@ impl HelixGraphStorage { .create(&mut wtxn)?; let mut secondary_indices = HashMap::new(); - if let Some(indexes) = config.graph_config.secondary_indices { + if let Some(indexes) = config.get_graph_config().secondary_indices { for index in indexes { secondary_indices.insert( index.clone(), @@ -127,18 +127,19 @@ impl HelixGraphStorage { } } + let vector_config = config.get_vector_config(); let vectors = VectorCore::new( &graph_env, &mut wtxn, HNSWConfig::new( - config.vector_config.m, - config.vector_config.ef_construction, - config.vector_config.ef_search, + vector_config.m, + vector_config.ef_construction, + vector_config.ef_search, ), )?; let bm25 = config - .bm25 + .get_bm25() .then(|| HBM25Config::new(&graph_env, &mut wtxn)) .transpose()?; @@ -400,4 +401,3 @@ impl StorageMethods for HelixGraphStorage { Ok(()) } } - diff --git a/helix-db/src/helix_engine/types.rs b/helix-db/src/helix_engine/types.rs index 207a23e0..fba07d79 100644 --- a/helix-db/src/helix_engine/types.rs +++ b/helix-db/src/helix_engine/types.rs @@ -141,6 +141,7 @@ pub enum VectorError { EntryPointNotFound, ConversionError(String), VectorCoreError(String), + VectorAlreadyDeleted(String), } impl fmt::Display for VectorError { @@ -152,6 +153,7 @@ impl fmt::Display for VectorError { VectorError::EntryPointNotFound => write!(f, "Entry point not found"), VectorError::ConversionError(msg) => write!(f, "Conversion error: {}", msg), VectorError::VectorCoreError(msg) => write!(f, "Vector core error: {}", msg), + VectorError::VectorAlreadyDeleted(id) => write!(f, "Vector already deleted: {}", id), } } } diff --git a/helix-db/src/helix_engine/vector_core/hnsw.rs b/helix-db/src/helix_engine/vector_core/hnsw.rs index 46b747dc..9d3c78eb 100644 --- a/helix-db/src/helix_engine/vector_core/hnsw.rs +++ b/helix-db/src/helix_engine/vector_core/hnsw.rs @@ -61,6 +61,18 @@ pub trait HNSW level: Option, ) -> Result, VectorError>; + /// Delete a vector from the index + /// + /// # Arguments + /// + /// * `txn` - The transaction to use + /// * `id` - The id of the vector + fn delete( + &self, + txn: &mut RwTxn, + id: u128, + ) -> Result<(), VectorError>; + /// Get specific vector based on id and level /// /// # Arguments diff --git a/helix-db/src/helix_engine/vector_core/mod.rs b/helix-db/src/helix_engine/vector_core/mod.rs index 2cbeb132..ecbf22a4 100644 --- a/helix-db/src/helix_engine/vector_core/mod.rs +++ b/helix-db/src/helix_engine/vector_core/mod.rs @@ -1,5 +1,6 @@ -pub mod vector; pub mod hnsw; +pub mod utils; +pub mod vector; pub mod vector_core; pub mod vector_distance; diff --git a/helix-db/src/helix_engine/vector_core/utils.rs b/helix-db/src/helix_engine/vector_core/utils.rs new file mode 100644 index 00000000..a48ca658 --- /dev/null +++ b/helix-db/src/helix_engine/vector_core/utils.rs @@ -0,0 +1,145 @@ +use std::{cmp::Ordering, collections::BinaryHeap}; + +use heed3::RoTxn; + +use crate::{protocol::value::Value, utils::filterable::Filterable}; + +#[derive(PartialEq)] +pub(super) struct Candidate { + pub id: u128, + pub distance: f64, +} + +impl Eq for Candidate {} + +impl PartialOrd for Candidate { + fn partial_cmp(&self, other: &Self) -> Option { + other.distance.partial_cmp(&self.distance) + } +} + +impl Ord for Candidate { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap_or(Ordering::Equal) + } +} + +pub(super) trait HeapOps { + /// Extend the heap with another heap + /// Used because using `.extend()` does not keep the order + fn extend_inord(&mut self, other: BinaryHeap) + where + T: Ord; + + /// Take the top k elements from the heap + /// Used because using `.iter()` does not keep the order + fn take_inord(&mut self, k: usize) -> BinaryHeap + where + T: Ord; + + /// Take the top k elements from the heap and return a vector + fn to_vec(&mut self, k: usize) -> Vec + where + T: Ord; + + /// Get the maximum element from the heap + fn get_max(&self) -> Option<&T> + where + T: Ord; + + fn to_vec_with_filter( + &mut self, + k: usize, + filter: Option<&[F]>, + txn: &RoTxn, + ) -> Vec + where + T: Ord + Filterable, + F: Fn(&T, &RoTxn) -> bool; +} + +impl HeapOps for BinaryHeap { + #[inline(always)] + fn extend_inord(&mut self, mut other: BinaryHeap) + where + T: Ord, + { + self.reserve(other.len()); + for item in other.drain() { + self.push(item); + } + } + + #[inline(always)] + fn take_inord(&mut self, k: usize) -> BinaryHeap + where + T: Ord, + { + let mut result = BinaryHeap::with_capacity(k); + for _ in 0..k { + if let Some(item) = self.pop() { + result.push(item); + } else { + break; + } + } + result + } + + #[inline(always)] + fn to_vec(&mut self, k: usize) -> Vec + where + T: Ord, + { + let mut result = Vec::with_capacity(k); + for _ in 0..k { + if let Some(item) = self.pop() { + result.push(item); + } else { + break; + } + } + result + } + + #[inline(always)] + fn get_max(&self) -> Option<&T> + where + T: Ord, + { + self.iter().max() + } + + #[inline(always)] + fn to_vec_with_filter( + &mut self, + k: usize, + filter: Option<&[F]>, + txn: &RoTxn, + ) -> Vec + where + T: Ord + Filterable, + F: Fn(&T, &RoTxn) -> bool, + { + let mut result = Vec::with_capacity(k); + for _ in 0..k { + // while pop check filters and pop until one passes + while let Some(item) = self.pop() { + if SHOULD_CHECK_DELETED { + if let Ok(is_deleted) = item.check_property("is_deleted") { + if let Value::Boolean(is_deleted) = is_deleted.as_ref() { + if *is_deleted { + continue; + } + } + } + } + if filter.is_none() || filter.unwrap().iter().all(|f| f(&item, txn)) { + result.push(item); + break; + } + } + } + result + } +} diff --git a/helix-db/src/helix_engine/vector_core/vector.rs b/helix-db/src/helix_engine/vector_core/vector.rs index 49e3abd5..1a642e37 100644 --- a/helix-db/src/helix_engine/vector_core/vector.rs +++ b/helix-db/src/helix_engine/vector_core/vector.rs @@ -3,15 +3,15 @@ use crate::{ types::{GraphError, VectorError}, vector_core::vector_distance::DistanceCalc, }, - protocol::{ - return_values::ReturnValue, - value::Value, + protocol::{return_values::ReturnValue, value::Value}, + utils::{ + filterable::{Filterable, FilterableType}, + id::v6_uuid, }, - utils::{filterable::{Filterable, FilterableType}, id::v6_uuid}, }; use core::fmt; use serde::{Deserialize, Serialize}; -use std::{cmp::Ordering, collections::HashMap, fmt::Debug}; +use std::{borrow::Cow, cmp::Ordering, collections::HashMap, fmt::Debug}; // TODO: make this generic over the type of encoding (f32, f64, etc) // TODO: use const param to set dimension @@ -23,7 +23,7 @@ pub struct HVector { /// The id of the HVector pub id: u128, /// Whether the HVector is deleted (will be used for soft deletes) - pub is_deleted: bool, + // pub is_deleted: bool, /// The level of the HVector pub level: usize, /// The distance of the HVector @@ -48,7 +48,16 @@ impl Ord for HVector { impl Debug for HVector { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{{ \nid: {},\nis_deleted: {},\nlevel: {},\ndistance: {:?},\ndata: {:?},\nproperties: {:#?} }}", uuid::Uuid::from_u128(self.id).to_string(), self.is_deleted, self.level, self.distance, self.data, self.properties) + write!( + f, + "{{ \nid: {},\nlevel: {},\ndistance: {:?},\ndata: {:?},\nproperties: {:#?} }}", + uuid::Uuid::from_u128(self.id).to_string(), + // self.is_deleted, + self.level, + self.distance, + self.data, + self.properties + ) } } @@ -58,7 +67,7 @@ impl HVector { let id = v6_uuid(); HVector { id, - is_deleted: false, + // is_deleted: false, level: 0, data, distance: None, @@ -71,7 +80,7 @@ impl HVector { let id = v6_uuid(); HVector { id, - is_deleted: false, + // is_deleted: false, level, data, distance: None, @@ -125,7 +134,7 @@ impl HVector { Ok(HVector { id, - is_deleted: false, + // is_deleted: false, level, data, distance: None, @@ -164,11 +173,10 @@ impl HVector { } } - /// Filterable implementation for HVector -/// +/// /// see helix_db/src/protocol/filterable.rs -/// +/// /// NOTE: This could be moved to the protocol module with the node and edges in `helix_db/protocol/items.rs`` impl Filterable for HVector { fn type_name(&self) -> FilterableType { @@ -237,18 +245,26 @@ impl Filterable for HVector { &self.properties } - fn check_property(&self, key: &str) -> Result<&Value, GraphError> { - match &self.properties { - Some(properties) => properties - .get(key) - .ok_or(GraphError::ConversionError(format!( + fn check_property(&self, key: &str) -> Result, GraphError> { + match key { + "id" => Ok(Cow::Owned(Value::from(self.id))), + "label" => Ok(Cow::Owned(Value::from(self.label().to_string()))), + "data" => Ok(Cow::Owned(Value::Array( + self.data.iter().map(|f| Value::F64(*f)).collect(), + ))), + _ => match &self.properties { + Some(properties) => properties + .get(key) + .ok_or(GraphError::ConversionError(format!( + "Property {} not found", + key + ))) + .map(|v| Cow::Borrowed(v)), + None => Err(GraphError::ConversionError(format!( "Property {} not found", key ))), - None => Err(GraphError::ConversionError(format!( - "Property {} not found", - key - ))), + }, } } diff --git a/helix-db/src/helix_engine/vector_core/vector_core.rs b/helix-db/src/helix_engine/vector_core/vector_core.rs index 3fcd045c..f0912949 100644 --- a/helix-db/src/helix_engine/vector_core/vector_core.rs +++ b/helix-db/src/helix_engine/vector_core/vector_core.rs @@ -1,18 +1,22 @@ use crate::helix_engine::{ types::VectorError, - vector_core::{hnsw::HNSW, vector::HVector}, + vector_core::{ + hnsw::HNSW, + utils::{Candidate, HeapOps}, + vector::HVector, + }, }; use crate::protocol::value::Value; use heed3::{ - types::{Bytes, Unit}, Database, Env, RoTxn, RwTxn, + types::{Bytes, Unit}, }; use itertools::Itertools; use rand::prelude::Rng; use serde::{Deserialize, Serialize}; use std::{ cmp::Ordering, - collections::{BinaryHeap, HashSet}, + collections::{BinaryHeap, HashMap, HashSet}, }; const DB_VECTORS: &str = "vectors"; // for vector data (v:) @@ -44,127 +48,6 @@ impl HNSWConfig { } } -#[derive(PartialEq)] -struct Candidate { - id: u128, - distance: f64, -} - -impl Eq for Candidate {} - -impl PartialOrd for Candidate { - fn partial_cmp(&self, other: &Self) -> Option { - other.distance.partial_cmp(&self.distance) - } -} - -impl Ord for Candidate { - fn cmp(&self, other: &Self) -> Ordering { - self.partial_cmp(other).unwrap_or(Ordering::Equal) - } -} - -pub trait HeapOps { - /// Extend the heap with another heap - /// Used because using `.extend()` does not keep the order - fn extend_inord(&mut self, other: BinaryHeap) - where - T: Ord; - - /// Take the top k elements from the heap - /// Used because using `.iter()` does not keep the order - fn take_inord(&mut self, k: usize) -> BinaryHeap - where - T: Ord; - - /// Take the top k elements from the heap and return a vector - fn to_vec(&mut self, k: usize) -> Vec - where - T: Ord; - - /// Get the maximum element from the heap - fn get_max(&self) -> Option<&T> - where - T: Ord; - - fn to_vec_with_filter(&mut self, k: usize, filter: Option<&[F]>, txn: &RoTxn) -> Vec - where - T: Ord, - F: Fn(&T, &RoTxn) -> bool; -} - -impl HeapOps for BinaryHeap { - #[inline(always)] - fn extend_inord(&mut self, mut other: BinaryHeap) - where - T: Ord, - { - self.reserve(other.len()); - for item in other.drain() { - self.push(item); - } - } - - #[inline(always)] - fn take_inord(&mut self, k: usize) -> BinaryHeap - where - T: Ord, - { - let mut result = BinaryHeap::with_capacity(k); - for _ in 0..k { - if let Some(item) = self.pop() { - result.push(item); - } else { - break; - } - } - result - } - - #[inline(always)] - fn to_vec(&mut self, k: usize) -> Vec - where - T: Ord, - { - let mut result = Vec::with_capacity(k); - for _ in 0..k { - if let Some(item) = self.pop() { - result.push(item); - } else { - break; - } - } - result - } - - #[inline(always)] - fn get_max(&self) -> Option<&T> - where - T: Ord, - { - self.iter().max() - } - - #[inline(always)] - fn to_vec_with_filter(&mut self, k: usize, filter: Option<&[F]>, txn: &RoTxn) -> Vec - where - T: Ord, - F: Fn(&T, &RoTxn) -> bool, - { - let mut result = Vec::with_capacity(k); - for _ in 0..k { - // while pop check filters and pop until one passes - while let Some(item) = self.pop() { - if filter.is_none() || filter.unwrap().iter().all(|f| f(&item, txn)) { - result.push(item); - break; - } - } - } - result - } -} - pub struct VectorCore { pub vectors_db: Database, pub vector_data_db: Database, @@ -533,7 +416,7 @@ impl HNSW for VectorCore { }, )?; - let mut results = candidates.to_vec_with_filter(k, filter, txn); + let mut results = candidates.to_vec_with_filter::(k, filter, txn); for result in results.iter_mut() { result.properties = match self @@ -631,6 +514,30 @@ impl HNSW for VectorCore { Ok(query) } + fn delete(&self, txn: &mut RwTxn, id: u128) -> Result<(), VectorError> { + let properties: Option> = + match self.vector_data_db.get(txn, &id.to_be_bytes())? { + Some(bytes) => Some(bincode::deserialize(&bytes).map_err(VectorError::from)?), + None => None, + }; + + println!("properties: {:?}", properties); + if let Some(mut properties) = properties { + if let Some(is_deleted) = properties.get("is_deleted") { + if let Value::Boolean(is_deleted) = is_deleted { + if *is_deleted { + return Err(VectorError::VectorAlreadyDeleted(id.to_string())); + } + } + } + properties.insert("is_deleted".to_string(), Value::Boolean(true)); + println!("properties: {:?}", properties); + self.vector_data_db + .put(txn, &id.to_be_bytes(), &bincode::serialize(&properties)?)?; + } + Ok(()) + } + fn get_all_vectors( &self, txn: &RoTxn, diff --git a/helix-db/src/helix_gateway/embedding_providers/embedding_providers.rs b/helix-db/src/helix_gateway/embedding_providers/embedding_providers.rs index e4092177..2d91d17f 100644 --- a/helix-db/src/helix_gateway/embedding_providers/embedding_providers.rs +++ b/helix-db/src/helix_gateway/embedding_providers/embedding_providers.rs @@ -152,7 +152,7 @@ pub fn get_embedding_model( /// ``` macro_rules! embed { ($db:expr, $query:expr) => {{ - let embedding_model = get_embedding_model(None, $db.embedding_model.as_deref(), None)?; + let embedding_model = get_embedding_model(None, $db.storage_config.embedding_model.as_deref(), None)?; embedding_model.fetch_embedding($query)? }}; ($db:expr, $query:expr, $model:expr) => {{ diff --git a/helix-db/src/helix_gateway/mcp/mcp.rs b/helix-db/src/helix_gateway/mcp/mcp.rs index 3587f1d6..c056ee80 100644 --- a/helix-db/src/helix_gateway/mcp/mcp.rs +++ b/helix-db/src/helix_gateway/mcp/mcp.rs @@ -198,7 +198,7 @@ pub fn collect<'a>(input: &'a mut MCPToolInput, response: &mut Response) -> Resu let (start, end) = match data.range { Some(range) => (range.start, range.end), - None => (0, 100), + None => (0, 10), }; let values = connection diff --git a/helix-db/src/helix_gateway/mcp/tools.rs b/helix-db/src/helix_gateway/mcp/tools.rs index 9a1eac11..2b678610 100644 --- a/helix-db/src/helix_gateway/mcp/tools.rs +++ b/helix-db/src/helix_gateway/mcp/tools.rs @@ -74,7 +74,7 @@ trait McpTools<'a> { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, edge_type: EdgeType, ) -> Result, GraphError>; @@ -82,14 +82,14 @@ trait McpTools<'a> { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, ) -> Result, GraphError>; fn in_step( &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, edge_type: EdgeType, ) -> Result, GraphError>; @@ -97,21 +97,21 @@ trait McpTools<'a> { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, ) -> Result, GraphError>; fn n_from_type( &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - node_type: &'a str, + node_type: String, ) -> Result, GraphError>; fn e_from_type( &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_type: &'a str, + edge_type: String, ) -> Result, GraphError>; /// filters items based on properies and traversal existence @@ -122,7 +122,6 @@ trait McpTools<'a> { connection: &'a MCPConnection, properties: Option>, filter_traversals: Option>, - _marker: PhantomData<&'a ()>, ) -> Result, GraphError>; /// BM25 @@ -148,7 +147,7 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, edge_type: EdgeType, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -157,7 +156,7 @@ impl<'a> McpTools<'a> for McpBackend { .iter .clone() .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); + let edge_label_hash = hash_label(&edge_label, None); let prefix = HelixGraphStorage::out_edge_key(&item.id(), &edge_label_hash); match db .out_edges_db @@ -189,7 +188,7 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -197,7 +196,7 @@ impl<'a> McpTools<'a> for McpBackend { .iter .clone() .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); + let edge_label_hash = hash_label(&edge_label, None); let prefix = HelixGraphStorage::out_edge_key(&item.id(), &edge_label_hash); match db .out_edges_db @@ -228,7 +227,7 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, edge_type: EdgeType, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -237,7 +236,7 @@ impl<'a> McpTools<'a> for McpBackend { .iter .clone() .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); + let edge_label_hash = hash_label(&edge_label, None); let prefix = HelixGraphStorage::in_edge_key(&item.id(), &edge_label_hash); match db .in_edges_db @@ -269,7 +268,7 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, txn: &'a RoTxn, connection: &'a MCPConnection, - edge_label: &'a str, + edge_label: String, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -277,7 +276,7 @@ impl<'a> McpTools<'a> for McpBackend { .iter .clone() .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); + let edge_label_hash = hash_label(&edge_label, None); let prefix = HelixGraphStorage::in_edge_key(&item.id(), &edge_label_hash); match db .in_edges_db @@ -308,13 +307,13 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, txn: &'a RoTxn, _connection: &'a MCPConnection, - node_type: &'a str, + node_type: String, ) -> Result, GraphError> { let db = Arc::clone(&self.db); let iter = NFromType { iter: db.nodes_db.lazily_decode_data().iter(txn).unwrap(), - label: node_type, + label: &node_type, }; let result = iter.take(100).collect::, _>>(); @@ -326,13 +325,13 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, txn: &'a RoTxn, _connection: &'a MCPConnection, - edge_type: &'a str, + edge_type: String, ) -> Result, GraphError> { let db = Arc::clone(&self.db); let iter = EFromType { iter: db.edges_db.lazily_decode_data().iter(txn).unwrap(), - label: edge_type, + label: &edge_type, }; let result = iter.take(100).collect::, _>>(); @@ -346,7 +345,6 @@ impl<'a> McpTools<'a> for McpBackend { connection: &'a MCPConnection, properties: Option>, filter_traversals: Option>, - _marker: PhantomData<&'a ()>, ) -> Result, GraphError> { let db = Arc::clone(&self.db); diff --git a/helix-db/src/helix_gateway/mod.rs b/helix-db/src/helix_gateway/mod.rs index cb65cbe4..dee78427 100644 --- a/helix-db/src/helix_gateway/mod.rs +++ b/helix-db/src/helix_gateway/mod.rs @@ -1,7 +1,6 @@ pub mod connection; +pub mod embedding_providers; pub mod gateway; +pub mod mcp; pub mod router; pub mod thread_pool; -pub mod mcp; -pub mod embedding_providers; - diff --git a/helix-db/src/helixc/analyzer/analyzer.rs b/helix-db/src/helixc/analyzer/analyzer.rs index 88845baf..f900845d 100644 --- a/helix-db/src/helixc/analyzer/analyzer.rs +++ b/helix-db/src/helixc/analyzer/analyzer.rs @@ -1,83 +1,20 @@ //! Semantic analyzer for Helix‑QL. -use super::{fix::Fix, pretty}; -use crate::{ - helix_engine::graph_core::ops::source::add_e::EdgeType, - helixc::{ - generator::{ - bool_op::{BoolOp, Eq, Gt, Gte, Lt, Lte, Neq}, - generator_types::{ - Assignment as GeneratedAssignment, BoExp, Drop as GeneratedDrop, ForEach as GeneratedForEach, ForLoopInVariable, ForVariable, Parameter as GeneratedParameter, Query as GeneratedQuery, ReturnType, ReturnValue, ReturnValueExpr, Source as GeneratedSource, Statement as GeneratedStatement - }, - object_remapping_generation::{ - ExcludeField, IdentifierRemapping, ObjectRemapping, Remapping, RemappingType, - TraversalRemapping, ValueRemapping, - }, - source_steps::{ - AddE, AddN, AddV, EFromID, EFromType, NFromID, NFromIndex, NFromType, SearchBM25, - SearchVector as GeneratedSearchVector, SourceStep, - }, - traversal_steps::{ - In as GeneratedIn, InE as GeneratedInE, OrderBy, Out as GeneratedOut, - OutE as GeneratedOutE, Range, SearchVectorStep, - ShortestPath as GeneratedShortestPath, ShouldCollect, Step as GeneratedStep, - Traversal as GeneratedTraversal, TraversalType, Where, WhereExists, WhereRef, - }, - utils::{GenRef, GeneratedValue, Order, Separator, VecData}, +use crate::helixc::{ + analyzer::{ + diagnostic::Diagnostic, + methods::{ + query_validation::validate_query, + schema_methods::{build_field_lookups, check_schema}, }, - parser::{helix_parser::*, location::Loc}, }, - protocol::{date::Date, value::Value}, - utils::styled_string::StyledString, + generator::generator_types::Source as GeneratedSource, + parser::helix_parser::{EdgeSchema, Field, Source}, }; use std::{ + borrow::Cow, collections::{HashMap, HashSet}, - convert::Infallible, }; -/// A single diagnostic to be surfaced to the editor. -#[derive(Debug, Clone)] -pub struct Diagnostic { - pub location: Loc, - pub message: String, - pub hint: Option, - pub filepath: Option, - pub severity: DiagnosticSeverity, - pub fix: Option, -} - -#[derive(Debug, Clone)] -pub enum DiagnosticSeverity { - Error, - Warning, - Info, - Hint, - Empty, -} - -impl Diagnostic { - pub fn new( - location: Loc, - message: impl Into, - severity: DiagnosticSeverity, - hint: Option, - fix: Option, - ) -> Self { - let filepath = location.filepath.clone(); - Self { - location, - message: message.into(), - hint, - fix, - filepath, - severity, - } - } - - pub fn render(&self, src: &str, filepath: &str) -> String { - pretty::render(self, src, filepath) - } -} - pub fn analyze(src: &Source) -> (Vec, GeneratedSource) { let mut ctx = Ctx::new(src); ctx.check_schema(); @@ -86,61 +23,24 @@ pub fn analyze(src: &Source) -> (Vec, GeneratedSource) { } /// Internal working context shared by all passes. -struct Ctx<'a> { - src: &'a Source, +pub(crate) struct Ctx<'a> { + pub(super) src: &'a Source, /// Quick look‑ups - node_set: HashSet<&'a str>, - vector_set: HashSet<&'a str>, - edge_map: HashMap<&'a str, &'a EdgeSchema>, - node_fields: HashMap<&'a str, HashMap<&'a str, &'a Field>>, - edge_fields: HashMap<&'a str, HashMap<&'a str, &'a Field>>, - vector_fields: HashMap<&'a str, HashMap<&'a str, &'a Field>>, - diagnostics: Vec, - output: GeneratedSource, + pub(super) node_set: HashSet<&'a str>, + pub(super) vector_set: HashSet<&'a str>, + pub(super) edge_map: HashMap<&'a str, &'a EdgeSchema>, + pub(super) node_fields: HashMap<&'a str, HashMap<&'a str, Cow<'a, Field>>>, + pub(super) edge_fields: HashMap<&'a str, HashMap<&'a str, Cow<'a, Field>>>, + pub(super) vector_fields: HashMap<&'a str, HashMap<&'a str, Cow<'a, Field>>>, + pub(super) diagnostics: Vec, + pub(super) output: GeneratedSource, } impl<'a> Ctx<'a> { - fn new(src: &'a Source) -> Self { + pub(super) fn new(src: &'a Source) -> Self { // Build field look‑ups once - let node_fields = src - .node_schemas - .iter() - .map(|n| { - ( - n.name.1.as_str(), - n.fields - .iter() - .map(|f| (f.name.as_str(), f)) - .collect::>(), - ) - }) - .collect(); - - let edge_fields = src - .edge_schemas - .iter() - .map(|e| { - ( - e.name.1.as_str(), - e.properties - .as_ref() - .map(|v| v.iter().map(|f| (f.name.as_str(), f)).collect()) - .unwrap_or_else(HashMap::new), - ) - }) - .collect(); - - let vector_fields = src - .vector_schemas - .iter() - .map(|v| { - ( - v.name.as_str(), - v.fields.iter().map(|f| (f.name.as_str(), f)).collect(), - ) - }) - .collect(); + let (node_fields, edge_fields, vector_fields) = build_field_lookups(src); let mut output = GeneratedSource::default(); output.src = src.source.clone(); @@ -164,4736 +64,14 @@ impl<'a> Ctx<'a> { // ---------- Pass #1: schema -------------------------- /// Validate that every edge references declared node types. - fn check_schema(&mut self) { - for edge in &self.src.edge_schemas { - if !self.node_set.contains(edge.from.1.as_str()) - && !self.vector_set.contains(edge.from.1.as_str()) - { - self.push_schema_err( - edge.from.0.clone(), - format!("`{}` is not a declared node type", edge.from.1), - Some(format!("Declare `N::{}` before this edge", edge.from.1)), - ); - } - if !self.node_set.contains(edge.to.1.as_str()) - && !self.vector_set.contains(edge.to.1.as_str()) - { - self.push_schema_err( - edge.to.0.clone(), - format!("`{}` is not a declared node type", edge.to.1), - Some(format!("Declare `N::{}` before this edge", edge.to.1)), - ); - } - edge.properties.as_ref().map(|v| { - v.iter().for_each(|f| { - if f.name.to_lowercase() == "id" { - self.push_schema_err( - f.loc.clone(), - format!("`{}` is a reserved field name", f.name), - Some("rename the field to something else".to_string()), - ); - } - }) - }); - self.output.edges.push(edge.clone().into()); - } - for node in &self.src.node_schemas { - node.fields.iter().for_each(|f| { - if f.name.to_lowercase() == "id" { - self.push_schema_err( - f.loc.clone(), - format!("`{}` is a reserved field name", f.name), - Some("rename the field to something else".to_string()), - ); - } - }); - self.output.nodes.push(node.clone().into()); - } - for vector in &self.src.vector_schemas { - vector.fields.iter().for_each(|f: &Field| { - if f.name.to_lowercase() == "id" { - self.push_schema_err( - f.loc.clone(), - format!("`{}` is a reserved field name", f.name), - Some("rename the field to something else".to_string()), - ); - } - }); - self.output.vectors.push(vector.clone().into()); - } + pub(super) fn check_schema(&mut self) { + check_schema(self); } // ---------- Pass #2: queries ------------------------- - fn check_queries(&mut self) { + pub(super) fn check_queries(&mut self) { for q in &self.src.queries { - self.check_query(q); - } - } - - fn check_query(&mut self, q: &'a Query) { - let mut query = GeneratedQuery::default(); - query.name = q.name.clone(); - // ------------------------------------------------- - // Parameter validation - // ------------------------------------------------- - for param in &q.parameters { - if let FieldType::Identifier(ref id) = param.param_type.1 { - if self.is_valid_identifier(q, param.param_type.0.clone(), id.as_str()) { - if !self.node_set.contains(id.as_str()) { - self.push_query_err( - q, - param.param_type.0.clone(), - format!("unknown type `{}` for parameter `{}`", id, param.name.1), - "declare or use a matching schema object or use a primitive type", - ); - } - } - } - // constructs parameters and sub‑parameters for generator - GeneratedParameter::unwrap_param( - param.clone(), - &mut query.parameters, - &mut query.sub_parameters, - ); - } - - // ------------------------------------------------- - // Statement‑by‑statement walk - // ------------------------------------------------- - let mut scope: HashMap<&str, Type> = HashMap::new(); - for param in &q.parameters { - scope.insert( - param.name.1.as_str(), - Type::from(param.param_type.1.clone()), - ); - } - for stmt in &q.statements { - let statement = self.walk_statements(&mut scope, q, &mut query, stmt); - if statement.is_some() { - query.statements.push(statement.unwrap()); - } else { - self.push_query_err( - q, - stmt.loc.clone(), - "invalid statement".to_string(), - "add a valid statement", - ); - } - } - - // ------------------------------------------------- - // Validate RETURN expressions - // ------------------------------------------------- - if q.return_values.is_empty() { - let end = q.loc.end.clone(); - self.push_query_warn( - q, - Loc::new(q.loc.filepath.clone(), end.clone(), end, q.loc.span.clone()), - "query has no RETURN clause".to_string(), - "add `RETURN ` at the end", - None, - ); - } - for ret in &q.return_values { - let (_, stmt) = self.infer_expr_type(ret, &mut scope, q, None, Some(&mut query)); - - assert!(stmt.is_some(), "RETURN value should be a valid expression"); - match stmt.unwrap() { - GeneratedStatement::Traversal(traversal) => match &traversal.source_step.inner() { - SourceStep::Identifier(v) => { - self.is_valid_identifier(q, ret.loc.clone(), v.inner().as_str()); - - // if is single object, need to handle it as a single object - // if is array, need to handle it as an array - match traversal.should_collect { - ShouldCollect::ToVec => { - query.return_values.push(ReturnValue::new_named( - GeneratedValue::Literal(GenRef::Literal(v.inner().clone())), - ReturnValueExpr::Traversal(traversal.clone()), - )); - } - ShouldCollect::ToVal => { - query.return_values.push(ReturnValue::new_single_named( - GeneratedValue::Literal(GenRef::Literal(v.inner().clone())), - ReturnValueExpr::Traversal(traversal.clone()), - )); - } - _ => { - unreachable!() - } - } - } - _ => { - query.return_values.push(ReturnValue::new_unnamed( - ReturnValueExpr::Traversal(traversal.clone()), - )); - } - }, - GeneratedStatement::Identifier(id) => { - self.is_valid_identifier(q, ret.loc.clone(), id.inner().as_str()); - let identifier_end_type = match scope.get(id.inner().as_str()) { - Some(t) => t.clone(), - None => { - self.push_query_err( - q, - ret.loc.clone(), - format!("variable named `{}` is not in scope", id), - "declare it earlier or fix the typo", - ); - Type::Unknown - } - }; - let value = self.gen_identifier_or_param(q, id.inner().as_str(), false, true); - match identifier_end_type { - Type::Scalar(_) => { - query.return_values.push(ReturnValue::new_named_literal( - GeneratedValue::Literal(GenRef::Literal(id.inner().clone())), - value, - )); - } - Type::Node(_) | Type::Vector(_) | Type::Edge(_) => { - query.return_values.push(ReturnValue::new_single_named( - GeneratedValue::Literal(GenRef::Literal(id.inner().clone())), - ReturnValueExpr::Identifier(value), - )); - } - _ => { - query.return_values.push(ReturnValue::new_named( - GeneratedValue::Literal(GenRef::Literal(id.inner().clone())), - ReturnValueExpr::Identifier(value), - )); - } - } - } - GeneratedStatement::Literal(l) => { - query.return_values.push(ReturnValue::new_literal( - GeneratedValue::Literal(l.clone()), - GeneratedValue::Literal(l.clone()), - )); - } - GeneratedStatement::Empty => query.return_values = vec![], - _ => { - self.push_query_err( - q, - ret.loc.clone(), - "RETURN value is not a valid expression".to_string(), - "add a valid expression", - ); - } - } - } - if q.is_mcp { - if query.return_values.len() != 1 { - self.push_query_err( - q, - q.loc.clone(), - "MCP queries can only return a single value as LLM needs to be able to traverse from the result".to_string(), - "add a single return value that is a node, edge, or vector", - ); - } else { - // match query.return_values.first().unwrap().return_type { - - // } - } - let return_name = query.return_values.first().unwrap().get_name(); - query.mcp_handler = Some(return_name); - } - self.output.queries.push(query); - } - - // ----------------------------------------------------- - // Helpers - // ----------------------------------------------------- - fn push_schema_err(&mut self, loc: Loc, msg: String, hint: Option) { - self.diagnostics.push(Diagnostic::new( - loc, - msg, - DiagnosticSeverity::Error, - hint, - None, - )); - } - fn push_query_err(&mut self, q: &Query, loc: Loc, msg: String, hint: impl Into) { - self.diagnostics.push(Diagnostic::new( - Loc::new(q.loc.filepath.clone(), loc.start, loc.end, loc.span), - format!("{} (in QUERY named `{}`)", msg, q.name), - DiagnosticSeverity::Error, - Some(hint.into()), - None, - )); - } - - fn push_query_err_with_fix( - &mut self, - q: &Query, - loc: Loc, - msg: String, - hint: impl Into, - fix: Fix, - ) { - self.diagnostics.push(Diagnostic::new( - Loc::new(q.loc.filepath.clone(), loc.start, loc.end, loc.span), - format!("{} (in QUERY named `{}`)", msg, q.name), - DiagnosticSeverity::Error, - Some(hint.into()), - Some(fix), - )); - } - - fn push_query_warn( - &mut self, - q: &Query, - loc: Loc, - msg: String, - hint: impl Into, - fix: Option, - ) { - self.diagnostics.push(Diagnostic::new( - Loc::new(q.loc.filepath.clone(), loc.start, loc.end, loc.span), - format!("{} (in QUERY named `{}`)", msg, q.name), - DiagnosticSeverity::Warning, - Some(hint.into()), - fix, - )); - } - - /// Infer the semantic `Type` of an expression *and* perform all traversal - ///‑specific validations (property access, field exclusions, step ordering). - fn infer_expr_type( - &mut self, - expression: &'a Expression, - scope: &mut HashMap<&'a str, Type>, - q: &'a Query, - parent_ty: Option, - gen_query: Option<&mut GeneratedQuery>, - ) -> (Type, Option) { - // TODO: Look at returning statement as well or passing mut query to push to - use ExpressionType::*; - let expr = &expression.expr; - match expr { - Identifier(name) => { - self.is_valid_identifier(q, expression.loc.clone(), name.as_str()); - match scope.get(name.as_str()) { - Some(t) => ( - t.clone(), - Some(GeneratedStatement::Identifier(GenRef::Std(name.clone()))), - ), - - None => { - self.push_query_err( - q, - expression.loc.clone(), - format!("variable named `{}` is not in scope", name), - "declare it earlier or fix the typo", - ); - (Type::Unknown, None) - } - } - } - - IntegerLiteral(i) => ( - Type::Scalar(FieldType::I32), - Some(GeneratedStatement::Literal(GenRef::Literal(i.to_string()))), - ), - FloatLiteral(f) => ( - Type::Scalar(FieldType::F64), - Some(GeneratedStatement::Literal(GenRef::Literal(f.to_string()))), - ), - StringLiteral(s) => ( - Type::Scalar(FieldType::String), - Some(GeneratedStatement::Literal(GenRef::Literal(s.to_string()))), - ), - BooleanLiteral(b) => ( - Type::Boolean, - Some(GeneratedStatement::Literal(GenRef::Literal(b.to_string()))), - ), - - Traversal(tr) => { - let mut gen_traversal = GeneratedTraversal::default(); - let final_ty = - self.check_traversal(tr, scope, q, parent_ty, &mut gen_traversal, gen_query); - // push query - let stmt = GeneratedStatement::Traversal(gen_traversal); - - if matches!(expr, Exists(_)) { - (Type::Boolean, Some(stmt)) - } else { - (final_ty, Some(stmt)) - } - } - - AddNode(add) => { - if let Some(ref ty) = add.node_type { - if !self.node_set.contains(ty.as_str()) { - self.push_query_err( - q, - add.loc.clone(), - format!("`AddN<{}>` refers to unknown node type", ty), - "declare the node schema first", - ); - } - let label = GenRef::Literal(ty.clone()); - - let node_in_schema = self - .output - .nodes - .iter() - .find(|n| n.name == ty.as_str()) - .unwrap() - .clone(); - - // Validate fields if both type and fields are present - if let Some(fields) = &add.fields { - // Get the field set before validation - // TODO: Check field types - let field_set = self.node_fields.get(ty.as_str()).cloned(); - if let Some(field_set) = field_set { - for (field_name, value) in fields { - if !field_set.contains_key(field_name.as_str()) { - self.push_query_err( - q, - add.loc.clone(), - format!("`{}` is not a field of node `{}`", field_name, ty), - "check the schema field names", - ); - } - match value { - ValueType::Identifier { value, loc } => { - if self.is_valid_identifier(q, loc.clone(), value.as_str()) - { - if !scope.contains_key(value.as_str()) { - self.push_query_err( - q, - loc.clone(), - format!("`{}` is not in scope", value), - "declare it earlier or fix the typo", - ); - } - }; - } - ValueType::Literal { value, loc } => { - // check against type - let field_type = self - .node_fields - .get(ty.as_str()) - .unwrap() - .get(field_name.as_str()) - .unwrap() - .field_type - .clone(); - if field_type != *value { - self.push_query_err( - q, - loc.clone(), - format!("value `{}` is of type `{}`, which does not match type {} declared in the schema for field `{}` on node type `{}`", value, GenRef::from(value.clone()), field_type, field_name, ty), - "ensure the value is of the same type as the field declared in the schema".to_string(), - ); - } - } - _ => {} - } - } - } - let mut properties: HashMap = fields - .iter() - .map(|(field_name, value)| { - ( - field_name.clone(), - match value { - ValueType::Literal { value, loc } => { - match self - .node_fields - .get(ty.as_str()) - .unwrap() - .get(field_name.as_str()) - .unwrap() - .field_type - == FieldType::Date - { - true => match Date::new(value) { - Ok(date) => GeneratedValue::Literal( - GenRef::Literal(date.to_rfc3339()), - ), - Err(e) => { - self.push_query_err( - q, - loc.clone(), - e.to_string(), - "ensure the value is a valid date", - ); - GeneratedValue::Unknown - } - }, - false => GeneratedValue::Literal(GenRef::from( - value.clone(), - )), - } - } - ValueType::Identifier { value, loc } => { - self.is_valid_identifier( - q, - loc.clone(), - value.as_str(), - ); - self.gen_identifier_or_param(q, value, true, false) - } - v => { - self.push_query_err( - q, - add.loc.clone(), - format!("`{:?}` is not a valid field value", v), - "use a literal or identifier", - ); - GeneratedValue::Unknown - } - }, - ) - }) - .collect(); - - let default_properties = node_in_schema - .properties - .iter() - .filter_map(|p| p.default_value.clone().map(|v| (p.name.clone(), v))) - .collect::>(); - - for (field_name, default_value) in default_properties { - if !properties.contains_key(field_name.as_str()) { - properties.insert(field_name, default_value); - } - } - - let secondary_indices = { - let secondary_indices = node_in_schema - .properties - .iter() - .filter_map(|p| { - matches!(p.is_index, FieldPrefix::Index) - .then_some(p.name.clone()) - }) - .collect::>(); - match secondary_indices.is_empty() { - true => None, - false => Some(secondary_indices), - } - }; - - let add_n = AddN { - label, - properties: Some(properties.into_iter().collect()), - secondary_indices, - }; - - let stmt = GeneratedStatement::Traversal(GeneratedTraversal { - source_step: Separator::Period(SourceStep::AddN(add_n)), - steps: vec![], - traversal_type: TraversalType::Mut, - should_collect: ShouldCollect::ToVal, - }); - if let Some(gen_query) = gen_query { - gen_query.is_mut = true; - } - return (Type::Node(Some(ty.to_string())), Some(stmt)); - } - } - self.push_query_err( - q, - add.loc.clone(), - "`AddN` must have a node type".to_string(), - "add a node type", - ); - return (Type::Node(None), None); - } - AddEdge(add) => { - if let Some(ref ty) = add.edge_type { - if !self.edge_map.contains_key(ty.as_str()) { - self.push_query_err( - q, - add.loc.clone(), - format!("`AddE<{}>` refers to unknown edge type", ty), - "declare the edge schema first", - ); - } - let label = GenRef::Literal(ty.clone()); - // Validate fields if both type and fields are present - let properties = match &add.fields { - Some(fields) => { - // Get the field set before validation - let field_set = self.edge_fields.get(ty.as_str()).cloned(); - if let Some(field_set) = field_set { - for (field_name, value) in fields { - if !field_set.contains_key(field_name.as_str()) { - self.push_query_err( - q, - add.loc.clone(), - format!( - "`{}` is not a field of edge `{}`", - field_name, ty - ), - "check the schema field names", - ); - } - - match value { - ValueType::Identifier { value, loc } => { - if self.is_valid_identifier( - q, - loc.clone(), - value.as_str(), - ) { - if !scope.contains_key(value.as_str()) { - self.push_query_err( - q, - loc.clone(), - format!("`{}` is not in scope", value), - "declare it earlier or fix the typo", - ); - } - }; - } - ValueType::Literal { value, loc } => { - // check against type - let field_type = self - .edge_fields - .get(ty.as_str()) - .unwrap() - .get(field_name.as_str()) - .unwrap() - .field_type - .clone(); - if field_type != *value { - self.push_query_err( - q, - loc.clone(), - format!("value `{}` is of type `{}`, which does not match type {} declared in the schema for field `{}` on node type `{}`", value, GenRef::from(value.clone()), field_type, field_name, ty), - "ensure the value is of the same type as the field declared in the schema".to_string(), - ); - } - } - _ => {} - } - } - } - Some( - fields - .iter() - .map(|(field_name, value)| { - ( - field_name.clone(), - match value { - ValueType::Literal { value, loc } => { - match self - .edge_fields - .get(ty.as_str()) - .unwrap() - .get(field_name.as_str()) - .unwrap() - .field_type - == FieldType::Date - { - true => match Date::new(value) { - Ok(date) => GeneratedValue::Literal( - GenRef::Literal(date.to_rfc3339()), - ), - Err(e) => { - self.push_query_err( - q, - loc.clone(), - e.to_string(), - "ensure the value is a valid date", - ); - GeneratedValue::Unknown - } - }, - false => GeneratedValue::Literal(GenRef::from( - value.clone(), - )), - } - } - ValueType::Identifier { value, loc } => { - self.is_valid_identifier( - q, - loc.clone(), - value.as_str(), - ); - self.gen_identifier_or_param(q, value.as_str(), false, true) - } - v => { - self.push_query_err( - q, - add.loc.clone(), - format!( - "`{:?}` is not a valid field value", - v - ), - "use a literal or identifier", - ); - GeneratedValue::Unknown - } - }, - ) - }) - .collect(), - ) - } - None => None, - }; - - let to = match &add.connection.to_id { - Some(id) => match id { - IdType::Identifier { value, loc } => { - self.is_valid_identifier(q, loc.clone(), value.as_str()); - self.gen_id_access_or_param(q, value.as_str()) - } - IdType::Literal { value, loc } => { - GeneratedValue::Literal(GenRef::Literal(value.clone())) - } - _ => unreachable!(), - }, - _ => { - self.push_query_err( - q, - add.loc.clone(), - "`AddE` must have a to id".to_string(), - "add a to id", - ); - GeneratedValue::Unknown - } - }; - let from = match &add.connection.from_id { - Some(id) => match id { - IdType::Identifier { value, loc } => { - self.is_valid_identifier(q, loc.clone(), value.as_str()); - self.gen_id_access_or_param(q, value.as_str()) - } - IdType::Literal { value, loc } => { - GeneratedValue::Literal(GenRef::Literal(value.clone())) - } - _ => unreachable!(), - }, - _ => { - self.push_query_err( - q, - add.loc.clone(), - "`AddE` must have a from id".to_string(), - "add a from id", - ); - GeneratedValue::Unknown - } - }; - let add_e = AddE { - to, - from, - label, - properties, - // secondary_indices: None, // TODO: Add secondary indices by checking against labeled `INDEX` fields in schema - }; - let stmt = GeneratedStatement::Traversal(GeneratedTraversal { - source_step: Separator::Period(SourceStep::AddE(add_e)), - steps: vec![], - traversal_type: TraversalType::Mut, - should_collect: ShouldCollect::ToVal, - }); - if let Some(gen_query) = gen_query { - gen_query.is_mut = true; - } - return (Type::Edge(Some(ty.to_string())), Some(stmt)); - } - self.push_query_err( - q, - add.loc.clone(), - "`AddE` must have an edge type".to_string(), - "add an edge type", - ); - (Type::Edge(None), None) - } - AddVector(add) => { - if let Some(ref ty) = add.vector_type { - if !self.vector_set.contains(ty.as_str()) { - self.push_query_err( - q, - add.loc.clone(), - format!("vector type `{}` has not been declared", ty), - format!("add a `V::{}` schema first", ty), - ); - } - // Validate vector fields - let (label, properties) = match &add.fields { - Some(fields) => { - let field_set = self.vector_fields.get(ty.as_str()).cloned(); - if let Some(field_set) = field_set { - for (field_name, value) in fields { - if !field_set.contains_key(field_name.as_str()) { - self.push_query_err( - q, - add.loc.clone(), - format!( - "`{}` is not a field of vector `{}`", - field_name, ty - ), - "check the schema field names", - ); - } - match value { - ValueType::Identifier { value, loc } => { - if self.is_valid_identifier( - q, - loc.clone(), - value.as_str(), - ) { - if !scope.contains_key(value.as_str()) { - self.push_query_err( - q, - loc.clone(), - format!("`{}` is not in scope", value), - "declare it earlier or fix the typo", - ); - } - }; - } - ValueType::Literal { value, loc } => { - // check against type - let field_type = self - .vector_fields - .get(ty.as_str()) - .unwrap() - .get(field_name.as_str()) - .unwrap() - .field_type - .clone(); - if field_type != *value { - self.push_query_err( - q, - loc.clone(), - format!("value `{}` is of type `{}`, which does not match type {} declared in the schema for field `{}` on node type `{}`", value, GenRef::from(value.clone()), field_type, field_name, ty), - "ensure the value is of the same type as the field declared in the schema".to_string(), - ); - } - } - _ => {} - } - } - } - let label = GenRef::Literal(ty.clone()); - let properties = fields - .iter() - .map(|(field_name, value)| { - ( - field_name.clone(), - match value { - ValueType::Literal { value, loc } => { - match self - .vector_fields - .get(ty.as_str()) - .unwrap() - .get(field_name.as_str()) - .unwrap() - .field_type - == FieldType::Date - { - true => match Date::new(value) { - Ok(date) => GeneratedValue::Literal( - GenRef::Literal(date.to_rfc3339()), - ), - Err(e) => { - self.push_query_err( - q, - loc.clone(), - e.to_string(), - "ensure the value is a valid date", - ); - GeneratedValue::Unknown - } - }, - false => GeneratedValue::Literal(GenRef::from( - value.clone(), - )), - } - } - ValueType::Identifier { value, loc } => { - self.is_valid_identifier( - q, - loc.clone(), - value.as_str(), - ); - self.gen_identifier_or_param(q, value.as_str(), false, true) - } - v => { - self.push_query_err( - q, - add.loc.clone(), - format!("`{:?}` is not a valid field value", v), - "use a literal or identifier", - ); - GeneratedValue::Unknown - } - }, - ) - }) - .collect(); - (label, Some(properties)) - } - None => (GenRef::Literal(ty.clone()), None), - }; - if let Some(vec_data) = &add.data { - let vec = match vec_data { - VectorData::Vector(v) => { - VecData::Standard(GeneratedValue::Literal(GenRef::Ref(format!( - "[{}]", - v.iter() - .map(|f| f.to_string()) - .collect::>() - .join(",") - )))) - } - VectorData::Identifier(i) => { - self.is_valid_identifier(q, add.loc.clone(), i.as_str()); - let id = self.gen_identifier_or_param(q, i.as_str(), true, false); - VecData::Standard(id) - } - VectorData::Embed(e) => match &e.value { - EvaluatesToString::Identifier(i) => { - VecData::Embed(self.gen_identifier_or_param(q, i.as_str(), true, false)) - } - EvaluatesToString::StringLiteral(s) => { - VecData::Embed(GeneratedValue::Literal(GenRef::Ref(s.clone()))) - } - }, - }; - let add_v = AddV { - vec, - label, - properties, - }; - let stmt = GeneratedStatement::Traversal(GeneratedTraversal { - source_step: Separator::Period(SourceStep::AddV(add_v)), - steps: vec![], - traversal_type: TraversalType::Mut, - should_collect: ShouldCollect::ToVal, - }); - if let Some(gen_query) = gen_query { - gen_query.is_mut = true; - } - return (Type::Vector(Some(ty.to_string())), Some(stmt)); - } - } - self.push_query_err( - q, - add.loc.clone(), - "`AddV` must have a vector type".to_string(), - "add a vector type", - ); - (Type::Vector(None), None) - } - // BatchAddVector(add) => { - // if let Some(ref ty) = add.vector_type { - // if !self.vector_set.contains(ty.as_str()) { - // self.push_query_err( - // q, - // add.loc.clone(), - // format!("vector type `{}` has not been declared", ty), - // format!("add a `V::{}` schema first", ty), - // ); - // } - // } - // Type::Vector(add.vector_type.as_deref()) - // } - SearchVector(sv) => { - if let Some(ref ty) = sv.vector_type { - if !self.vector_set.contains(ty.as_str()) { - self.push_query_err( - q, - sv.loc.clone(), - format!("vector type `{}` has not been declared", ty), - format!("add a `V::{}` schema first", ty), - ); - } - } - let vec = match &sv.data { - Some(VectorData::Vector(v)) => GeneratedValue::Literal(GenRef::Ref(format!( - "[{}]", - v.iter() - .map(|f| f.to_string()) - .collect::>() - .join(",") - ))), - Some(VectorData::Identifier(i)) => { - self.is_valid_identifier(q, sv.loc.clone(), i.as_str()); - // if is in params then use data. - if let Some(_) = q.parameters.iter().find(|p| p.name.1 == *i) { - GeneratedValue::Identifier(GenRef::Ref(format!( - "data.{}", - i.to_string() - ))) - } else if let Some(_) = scope.get(i.as_str()) { - GeneratedValue::Identifier(GenRef::Ref(i.to_string())) - } else { - self.push_query_err( - q, - sv.loc.clone(), - format!("variable named `{}` is not in scope", i), - "declare {} in the current scope or fix the typo", - ); - GeneratedValue::Unknown - } - } - _ => { - self.push_query_err( - q, - sv.loc.clone(), - "`SearchVector` must have a vector data".to_string(), - "add a vector data", - ); - GeneratedValue::Unknown - } - }; - let k = match &sv.k { - Some(k) => match &k.value { - EvaluatesToNumberType::I8(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::I16(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::I32(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::I64(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - - EvaluatesToNumberType::U8(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U16(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U32(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U64(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U128(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::Identifier(i) => { - self.is_valid_identifier(q, sv.loc.clone(), i.as_str()); - // is param - if let Some(_) = q.parameters.iter().find(|p| p.name.1 == *i) { - GeneratedValue::Identifier(GenRef::Std(format!( - "data.{} as usize", - i - ))) - } else { - GeneratedValue::Identifier(GenRef::Std(i.to_string())) - } - } - _ => { - self.push_query_err( - q, - sv.loc.clone(), - "`SearchVector` must have a limit of vectors to return".to_string(), - "add a limit", - ); - GeneratedValue::Unknown - } - }, - None => { - self.push_query_err( - q, - sv.loc.clone(), - "`SearchV` must have a limit of vectors to return".to_string(), - "add a limit", - ); - GeneratedValue::Unknown - } - }; - - let pre_filter: Option> = match &sv.pre_filter { - Some(expr) => { - let (_, stmt) = self.infer_expr_type( - expr, - scope, - q, - Some(Type::Vector(sv.vector_type.clone())), - None, - ); - // Where/boolean ops don't change the element type, - // so `cur_ty` stays the same. - assert!(stmt.is_some()); - let stmt = stmt.unwrap(); - let mut gen_traversal = GeneratedTraversal { - traversal_type: TraversalType::NestedFrom(GenRef::Std("v".to_string())), - steps: vec![], - should_collect: ShouldCollect::ToVec, - source_step: Separator::Empty(SourceStep::Anonymous), - }; - match stmt { - GeneratedStatement::Traversal(tr) => { - gen_traversal - .steps - .push(Separator::Period(GeneratedStep::Where(Where::Ref( - WhereRef { - expr: BoExp::Expr(tr), - }, - )))); - } - GeneratedStatement::BoExp(expr) => { - gen_traversal - .steps - .push(Separator::Period(GeneratedStep::Where(match expr { - BoExp::Exists(tr) => Where::Exists(WhereExists { tr }), - _ => Where::Ref(WhereRef { expr }), - }))); - } - _ => unreachable!(), - } - Some(vec![BoExp::Expr(gen_traversal)]) - } - None => None, - }; - - // Search returns nodes that contain the vectors - ( - Type::Vectors(sv.vector_type.clone()), - Some(GeneratedStatement::Traversal(GeneratedTraversal { - traversal_type: TraversalType::Ref, - steps: vec![], - should_collect: ShouldCollect::ToVec, - source_step: Separator::Period(SourceStep::SearchVector( - GeneratedSearchVector { vec, k, pre_filter }, - )), - })), - ) - } - And(v) => { - let exprs = v - .iter() - .map(|expr| { - let (_, stmt) = - self.infer_expr_type(expr, scope, q, parent_ty.clone(), None); - assert!( - stmt.is_some(), - "incorrect stmt should've been caught by `infer_expr_type`" - ); - - match stmt.unwrap() { - GeneratedStatement::BoExp(expr) => { - match expr { - BoExp::Exists(mut tr) => { - // keep as iterator - tr.should_collect = ShouldCollect::No; - BoExp::Exists(tr) - } - _ => expr, - } - } - GeneratedStatement::Traversal(tr) => BoExp::Expr(tr), - _ => unreachable!(), - } - }) - .collect::>(); - ( - Type::Boolean, - Some(GeneratedStatement::BoExp(BoExp::And(exprs))), - ) - } - Or(v) => { - let exprs = v - .iter() - .map(|expr| { - let (_, stmt) = - self.infer_expr_type(expr, scope, q, parent_ty.clone(), None); - assert!( - stmt.is_some(), - "incorrect stmt should've been caught by `infer_expr_type`" - ); - match stmt.unwrap() { - GeneratedStatement::BoExp(expr) => match expr { - BoExp::Exists(mut tr) => { - tr.should_collect = ShouldCollect::No; - BoExp::Exists(tr) - } - _ => expr, - }, - GeneratedStatement::Traversal(tr) => BoExp::Expr(tr), - _ => unreachable!(), - } - }) - .collect::>(); - ( - Type::Boolean, - Some(GeneratedStatement::BoExp(BoExp::Or(exprs))), - ) - } - Exists(expr) => { - let (_, stmt) = self.infer_expr_type(expr, scope, q, parent_ty, gen_query); - assert!(stmt.is_some()); - assert!(matches!(stmt, Some(GeneratedStatement::Traversal(_)))); - let expr = match stmt.unwrap() { - GeneratedStatement::Traversal(mut tr) => { - tr.traversal_type = - TraversalType::NestedFrom(GenRef::Std("val".to_string())); - tr - } - _ => unreachable!(), - }; - ( - Type::Boolean, - Some(GeneratedStatement::BoExp(BoExp::Exists(expr))), - ) - } - Empty => (Type::Unknown, Some(GeneratedStatement::Empty)), - BM25Search(bm25_search) => { - // TODO: look into how best do type checking for type passed in - if let Some(ref ty) = bm25_search.type_arg { - if !self.node_set.contains(ty.as_str()) { - self.push_query_err( - q, - bm25_search.loc.clone(), - format!("vector type `{}` has not been declared", ty), - format!("add a `V::{}` schema first", ty), - ); - } - } - let vec = match &bm25_search.data { - Some(ValueType::Literal { value, loc }) => { - GeneratedValue::Literal(GenRef::Std(value.to_string())) - } - Some(ValueType::Identifier { value: i, loc }) => { - self.is_valid_identifier(q, bm25_search.loc.clone(), i.as_str()); - // if is in params then use data. - if let Some(_) = q.parameters.iter().find(|p| p.name.1 == *i) { - GeneratedValue::Identifier(GenRef::Ref(format!( - "data.{}", - i.to_string() - ))) - } else if let Some(_) = scope.get(i.as_str()) { - GeneratedValue::Identifier(GenRef::Ref(i.to_string())) - } else { - self.push_query_err( - q, - bm25_search.loc.clone(), - format!("variable named `{}` is not in scope", i), - "declare {} in the current scope or fix the typo", - ); - GeneratedValue::Unknown - } - } - _ => { - self.push_query_err( - q, - bm25_search.loc.clone(), - "`SearchVector` must have a vector data".to_string(), - "add a vector data", - ); - GeneratedValue::Unknown - } - }; - let k = match &bm25_search.k { - Some(k) => match &k.value { - EvaluatesToNumberType::I8(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::I16(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::I32(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::I64(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - - EvaluatesToNumberType::U8(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U16(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U32(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U64(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::U128(i) => { - GeneratedValue::Primitive(GenRef::Std(i.to_string())) - } - EvaluatesToNumberType::Identifier(i) => { - self.is_valid_identifier(q, bm25_search.loc.clone(), i.as_str()); - // is param - if let Some(_) = q.parameters.iter().find(|p| p.name.1 == *i) { - GeneratedValue::Identifier(GenRef::Std(format!( - "data.{} as usize", - i - ))) - } else { - GeneratedValue::Identifier(GenRef::Std(i.to_string())) - } - } - _ => { - self.push_query_err( - q, - bm25_search.loc.clone(), - "`SearchVector` must have a limit of vectors to return".to_string(), - "add a limit", - ); - GeneratedValue::Unknown - } - }, - None => { - self.push_query_err( - q, - bm25_search.loc.clone(), - "`SearchV` must have a limit of vectors to return".to_string(), - "add a limit", - ); - GeneratedValue::Unknown - } - }; - - let search_bm25 = SearchBM25 { - type_arg: GenRef::Literal(bm25_search.type_arg.clone().unwrap()), - query: vec, - k, - }; - ( - Type::Nodes(bm25_search.type_arg.clone()), - Some(GeneratedStatement::Traversal(GeneratedTraversal { - traversal_type: TraversalType::Ref, - steps: vec![], - should_collect: ShouldCollect::ToVec, - source_step: Separator::Period(SourceStep::SearchBM25(search_bm25)), - })), - ) - } - _ => { - println!("Unknown expression: {:?}", expr); - todo!() - } - } - } - - // ----------------------------------------------------- - // Traversal‑specific checks - // ----------------------------------------------------- - fn check_traversal( - &mut self, - tr: &'a Traversal, - scope: &mut HashMap<&'a str, Type>, - q: &'a Query, - parent_ty: Option, - gen_traversal: &mut GeneratedTraversal, - gen_query: Option<&mut GeneratedQuery>, - ) -> Type { - let mut previous_step = None; - let mut cur_ty = match &tr.start { - StartNode::Node { node_type, ids } => { - if !self.node_set.contains(node_type.as_str()) { - self.push_query_err( - q, - tr.loc.clone(), - format!("unknown node type `{}`", node_type), - format!("declare N::{} in the schema first", node_type), - ); - } - if let Some(ids) = ids { - assert!(ids.len() == 1, "multiple ids not supported yet"); - // check id exists in scope - match ids[0].clone() { - IdType::ByIndex { index, value, loc } => { - self.is_valid_identifier(q, loc.clone(), index.to_string().as_str()); - let corresponding_field = - self.node_fields.get(node_type.as_str()).cloned(); - match corresponding_field { - Some(node_fields) => { - match node_fields - .iter() - .find(|(name, _)| name.to_string() == *index.to_string()) - { - Some((_, field)) => { - if !field.is_indexed() { - self.push_query_err( - q, - loc.clone(), - format!("field `{}` has not been indexed for node type `{}`", index, node_type), - format!("use a field that has been indexed with `INDEX` in the schema for node type `{}`", node_type) - ); - } else { - if let ValueType::Literal { ref value, ref loc } = - *value - { - if !field.field_type.eq(value) { - self.push_query_err( - q, - loc.clone(), - format!("value `{}` is of type `{}`, expected `{}`", value.to_string(), value, field.field_type), - format!("use a value of type `{}`", field.field_type ) - ); - } - } - } - } - None => { - self.push_query_err( - q, - loc.clone(), - format!("field `{}` has not been defined and/or indexed for node type `{}`", index, node_type), - format!("use a field that has been defined and indexed with `INDEX` in the schema for node type `{}`", node_type) - ); - } - } - } - None => unreachable!(), - }; - gen_traversal.source_step = Separator::Period(SourceStep::NFromIndex( - NFromIndex { - index: GenRef::Literal(match *index { - IdType::Identifier { value, loc: _ } => value, - _ => { - self.push_query_err( - q, - loc.clone(), - "index type must be an identifier, got literal".to_string(), - "use an existing identifier from the shema that has been indexed with `INDEX` instead".to_string(), - ); - String::new() - } - }), - key: GenRef::Ref(match *value { - ValueType::Identifier { value: i, loc } => { - if self.is_valid_identifier(q, loc.clone(), i.as_str()) - { - if !scope.contains_key(i.as_str()) { - self.push_query_err( - q, - loc, - format!("variable named `{}` is not in scope", i), - format!( - "declare {} in the current scope or fix the typo", - i - ), - ); - } - } - format!("data.{}", i) - } - ValueType::Literal { value, loc } => match value { - Value::String(s) => format!("\"{}\"", s), - Value::I8(i) => i.to_string(), - Value::I16(i) => i.to_string(), - Value::I32(i) => i.to_string(), - Value::I64(i) => i.to_string(), - Value::U8(i) => i.to_string(), - Value::U16(i) => i.to_string(), - Value::U32(i) => i.to_string(), - Value::U64(i) => i.to_string(), - Value::U128(i) => i.to_string(), - Value::F32(i) => i.to_string(), - Value::F64(i) => i.to_string(), - Value::Boolean(b) => b.to_string(), - _ => unreachable!(), - }, - _ => unreachable!(), - }), - }, - )); - gen_traversal.should_collect = ShouldCollect::ToVal; - gen_traversal.traversal_type = TraversalType::Ref; - Type::Node(Some(node_type.to_string())) - } - IdType::Identifier { value: i, loc } => { - if self.is_valid_identifier(q, loc.clone(), i.as_str()) { - if !scope.contains_key(i.as_str()) { - self.push_query_err( - q, - loc.clone(), - format!("variable named `{}` is not in scope", i), - format!( - "declare {} in the current scope or fix the typo", - i - ), - ); - } - } - gen_traversal.source_step = - Separator::Period(SourceStep::NFromID(NFromID { - id: GenRef::Ref(format!("data.{}", i)), - label: GenRef::Literal(node_type.clone()), - })); - gen_traversal.traversal_type = TraversalType::Ref; - gen_traversal.should_collect = ShouldCollect::ToVal; - Type::Node(Some(node_type.to_string())) - } - IdType::Literal { value: s, loc } => { - gen_traversal.source_step = - Separator::Period(SourceStep::NFromID(NFromID { - id: GenRef::Ref(s), - label: GenRef::Literal(node_type.clone()), - })); - gen_traversal.traversal_type = TraversalType::Ref; - gen_traversal.should_collect = ShouldCollect::ToVal; - Type::Node(Some(node_type.to_string())) - } - } - } else { - gen_traversal.source_step = - Separator::Period(SourceStep::NFromType(NFromType { - label: GenRef::Literal(node_type.clone()), - })); - gen_traversal.traversal_type = TraversalType::Ref; - Type::Nodes(Some(node_type.to_string())) - } - } - StartNode::Edge { edge_type, ids } => { - if !self.edge_map.contains_key(edge_type.as_str()) { - self.push_query_err( - q, - tr.loc.clone(), - format!("unknown edge type `{}`", edge_type), - format!("declare E::{} in the schema first", edge_type), - ); - } - if let Some(ids) = ids { - assert!(ids.len() == 1, "multiple ids not supported yet"); - gen_traversal.source_step = Separator::Period(SourceStep::EFromID(EFromID { - id: match ids[0].clone() { - IdType::Identifier { value: i, loc } => { - if self.is_valid_identifier(q, loc.clone(), i.as_str()) { - if !scope.contains_key(i.as_str()) { - self.push_query_err( - q, - loc, - format!("variable named `{}` is not in scope", i), - format!( - "declare {} in the current scope or fix the typo", - i - ), - ); - } - } - GenRef::Std(format!("&data.{}", i)) - } - IdType::Literal { value: s, loc } => GenRef::Std(s), - _ => unreachable!(), - }, - label: GenRef::Literal(edge_type.clone()), - })); - gen_traversal.traversal_type = TraversalType::Ref; - Type::Edge(Some(edge_type.to_string())) - } else { - gen_traversal.source_step = - Separator::Period(SourceStep::EFromType(EFromType { - label: GenRef::Literal(edge_type.clone()), - })); - gen_traversal.traversal_type = TraversalType::Ref; - Type::Edges(Some(edge_type.to_string())) - } - } - - StartNode::Identifier(identifier) => { - match self.is_valid_identifier(q, tr.loc.clone(), identifier.as_str()) { - true => scope.get(identifier.as_str()).cloned().map_or_else( - || { - self.push_query_err( - q, - tr.loc.clone(), - format!("variable named `{}` is not in scope", identifier), - format!( - "declare {} in the current scope or fix the typo", - identifier - ), - ); - Type::Unknown - }, - |var_type| { - gen_traversal.traversal_type = - TraversalType::FromVar(GenRef::Std(identifier.clone())); - gen_traversal.source_step = Separator::Empty(SourceStep::Identifier( - GenRef::Std(identifier.clone()), - )); - var_type.clone() - }, - ), - false => Type::Unknown, - } - } - // anonymous will be the traversal type rather than the start type - StartNode::Anonymous => { - let parent = parent_ty.unwrap(); - gen_traversal.traversal_type = - TraversalType::FromVar(GenRef::Std("val".to_string())); // TODO: ensure this default is stable - gen_traversal.source_step = Separator::Empty(SourceStep::Anonymous); - parent - } - }; - - // Track excluded fields for property validation - let mut excluded: HashMap<&str, Loc> = HashMap::new(); - - // Stream through the steps - let number_of_steps = match tr.steps.len() { - 0 => 0, - n => n - 1, - }; - - for (i, graph_step) in tr.steps.iter().enumerate() { - let step = &graph_step.step; - match step { - StepType::Node(gs) | StepType::Edge(gs) => { - match self.apply_graph_step(&gs, &cur_ty, q, gen_traversal, scope) { - Some(new_ty) => { - cur_ty = new_ty; - } - None => { /* error already recorded */ } - } - excluded.clear(); // Traversal to a new element resets exclusions - } - - StepType::Count => { - cur_ty = Type::Scalar(FieldType::I64); - excluded.clear(); - gen_traversal - .steps - .push(Separator::Period(GeneratedStep::Count)); - gen_traversal.should_collect = ShouldCollect::No; - } - - StepType::Exclude(ex) => { - // checks if exclude is either the last step or the step before an object remapping or closure - // i.e. you cant have `N::!{field1}::Out