diff --git a/refact-agent/engine/Cargo.toml b/refact-agent/engine/Cargo.toml index 2c10b352f..49158c5f3 100644 --- a/refact-agent/engine/Cargo.toml +++ b/refact-agent/engine/Cargo.toml @@ -99,6 +99,7 @@ url = "2.4.1" uuid = { version = "1", features = ["v4", "serde"] } walkdir = "2.3" which = "7.0.1" +petgraph = "0.6" zerocopy = "0.8.14" # There you can use a local copy diff --git a/refact-agent/engine/src/agentic/generate_code_edit.rs b/refact-agent/engine/src/agentic/generate_code_edit.rs new file mode 100644 index 000000000..7c2cf7ffd --- /dev/null +++ b/refact-agent/engine/src/agentic/generate_code_edit.rs @@ -0,0 +1,163 @@ +use crate::at_commands::at_commands::AtCommandsContext; +use crate::call_validation::{ChatContent, ChatMessage}; +use crate::global_context::{try_load_caps_quickly_if_not_present, GlobalContext}; +use crate::subchat::subchat_single; +use std::sync::Arc; +use tokio::sync::Mutex as AMutex; +use tokio::sync::RwLock as ARwLock; + +const CODE_EDIT_SYSTEM_PROMPT: &str = r#"You are a code editing assistant. Your task is to modify the provided code according to the user's instruction. + +# Rules +1. Return ONLY the edited code - no explanations, no markdown fences, no commentary +2. Preserve the original indentation style and formatting conventions +3. Make minimal changes necessary to fulfill the instruction +4. If the instruction is unclear, make the most reasonable interpretation +5. Keep all code that isn't directly related to the instruction unchanged + +# Output Format +Return the edited code directly, without any wrapping or explanation. The output should be valid code that can directly replace the input."#; + +const N_CTX: usize = 32000; +const TEMPERATURE: f32 = 0.1; + +fn remove_markdown_fences(text: &str) -> String { + let trimmed = text.trim(); + if trimmed.starts_with("```") { + let lines: Vec<&str> = trimmed.lines().collect(); + if lines.len() >= 2 { + // Find closing fence + if let Some(end_idx) = lines.iter().rposition(|l| l.trim() == "```") { + if end_idx > 0 { + // Skip first line (```language) and last line (```) + let start_idx = 1; + if start_idx < end_idx { + return lines[start_idx..end_idx].join("\n"); + } + } + } + } + } + text.to_string() +} + +pub async fn generate_code_edit( + gcx: Arc>, + code: &str, + instruction: &str, + cursor_file: &str, + cursor_line: i32, +) -> Result { + if code.is_empty() { + return Err("The provided code is empty".to_string()); + } + if instruction.is_empty() { + return Err("The instruction is empty".to_string()); + } + + let user_message = format!( + "File: {} (line {})\n\nCode to edit:\n```\n{}\n```\n\nInstruction: {}", + cursor_file, cursor_line, code, instruction + ); + + let messages = vec![ + ChatMessage { + role: "system".to_string(), + content: ChatContent::SimpleText(CODE_EDIT_SYSTEM_PROMPT.to_string()), + ..Default::default() + }, + ChatMessage { + role: "user".to_string(), + content: ChatContent::SimpleText(user_message), + ..Default::default() + }, + ]; + + let model_id = match try_load_caps_quickly_if_not_present(gcx.clone(), 0).await { + Ok(caps) => { + // Prefer light model for fast inline edits, fallback to default + let light = &caps.defaults.chat_light_model; + if !light.is_empty() { + Ok(light.clone()) + } else { + Ok(caps.defaults.chat_default_model.clone()) + } + } + Err(_) => Err("No caps available".to_string()), + }?; + + let ccx: Arc> = Arc::new(AMutex::new( + AtCommandsContext::new( + gcx.clone(), + N_CTX, + 1, + false, + messages.clone(), + "".to_string(), + false, + model_id.clone(), + ) + .await, + )); + + let new_messages = subchat_single( + ccx.clone(), + &model_id, + messages, + Some(vec![]), // No tools - pure generation + None, + false, + Some(TEMPERATURE), + None, + 1, + None, + false, // Don't prepend system prompt - we have our own + None, + None, + None, + ) + .await + .map_err(|e| format!("Error generating code edit: {}", e))?; + + let edited_code = new_messages + .into_iter() + .next() + .and_then(|msgs| msgs.into_iter().last()) + .and_then(|msg| match msg.content { + ChatContent::SimpleText(text) => Some(text), + ChatContent::Multimodal(_) => None, + }) + .ok_or("No edited code was generated".to_string())?; + + // Strip markdown fences if present + Ok(remove_markdown_fences(&edited_code)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_remove_markdown_fences_with_language() { + let input = "```python\ndef hello():\n print('world')\n```"; + assert_eq!(remove_markdown_fences(input), "def hello():\n print('world')"); + } + + #[test] + fn test_remove_markdown_fences_without_language() { + let input = "```\nsome code\n```"; + assert_eq!(remove_markdown_fences(input), "some code"); + } + + #[test] + fn test_remove_markdown_fences_no_fences() { + let input = "plain code without fences"; + assert_eq!(remove_markdown_fences(input), "plain code without fences"); + } + + #[test] + fn test_remove_markdown_fences_with_whitespace() { + let input = " ```rust\nfn main() {}\n``` "; + assert_eq!(remove_markdown_fences(input), "fn main() {}"); + } +} diff --git a/refact-agent/engine/src/agentic/generate_commit_message.rs b/refact-agent/engine/src/agentic/generate_commit_message.rs index 2b788d756..cefa981ab 100644 --- a/refact-agent/engine/src/agentic/generate_commit_message.rs +++ b/refact-agent/engine/src/agentic/generate_commit_message.rs @@ -11,20 +11,64 @@ use tokio::sync::RwLock as ARwLock; use tracing::warn; use crate::files_in_workspace::detect_vcs_for_a_file_path; -const DIFF_ONLY_PROMPT: &str = r#"Analyze the given diff and generate a clear and descriptive commit message that explains the purpose of the changes. Your commit message should convey *why* the changes were made, *how* they improve the code, or what features or fixes are implemented, rather than just restating *what* the changes are. Aim for an informative, concise summary that would be easy for others to understand when reviewing the commit history. +const DIFF_ONLY_PROMPT: &str = r#"Generate a commit message following the Conventional Commits specification. -# Steps -1. Analyze the code diff to understand the changes made. -2. Determine the functionality added or removed, and the reason for these adjustments. -3. Summarize the details of the change in an accurate and informative, yet concise way. -4. Structure the message in a way that starts with a short summary line, followed by optional details if the change is complex. +# Conventional Commits Format + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +## Commit Types (REQUIRED - choose exactly one) +- `feat`: New feature (correlates with MINOR in SemVer) +- `fix`: Bug fix (correlates with PATCH in SemVer) +- `refactor`: Code restructuring without changing behavior +- `perf`: Performance improvement +- `docs`: Documentation only changes +- `style`: Code style changes (formatting, whitespace, semicolons) +- `test`: Adding or correcting tests +- `build`: Changes to build system or dependencies +- `ci`: Changes to CI configuration +- `chore`: Maintenance tasks (tooling, configs, no production code change) +- `revert`: Reverting a previous commit + +## Rules + +### Subject Line (REQUIRED) +1. Format: `(): ` or `: ` +2. Use imperative mood ("add" not "added" or "adds") +3. Do NOT capitalize the first letter of description +4. Do NOT end with a period +5. Keep under 50 characters (hard limit: 72) +6. Scope is optional but recommended for larger projects + +### Body (OPTIONAL - use for complex changes) +1. Separate from subject with a blank line +2. Wrap at 72 characters +3. Explain WHAT and WHY, not HOW +4. Use bullet points for multiple items + +### Footer (OPTIONAL) +1. Reference issues: `Fixes #123`, `Closes #456`, `Refs #789` +2. Breaking changes: Start with `BREAKING CHANGE:` or add `!` after type +3. Co-authors: `Co-authored-by: Name ` + +## Breaking Changes +- Add `!` after type/scope: `feat!:` or `feat(api)!:` +- Or include `BREAKING CHANGE:` footer with explanation -# Output Format +# Steps -The output should be a single commit message in the following format: -- A **first line summarizing** the purpose of the change. This line should be concise. -- Optionally, include a **second paragraph** with *additional context* if the change is complex or otherwise needs further clarification. - (e.g., if there's a bug fix, mention what problem was fixed and why the change works.) +1. Analyze the diff to understand what changed +2. Determine the PRIMARY type of change (feat, fix, refactor, etc.) +3. Identify scope from affected files/modules (optional) +4. Write description in imperative mood explaining the intent +5. Add body only if the change is complex and needs explanation +6. Add footer for issue references or breaking changes if applicable # Examples @@ -32,23 +76,17 @@ The output should be a single commit message in the following format: ```diff - public class UserManager { - private final UserDAO userDAO; - + public class UserManager { + private final UserService userService; + private final NotificationService notificationService; - - public UserManager(UserDAO userDAO) { -- this.userDAO = userDAO; -+ this.userService = new UserService(); -+ this.notificationService = new NotificationService(); - } ``` -**Output (commit message)**: +**Output**: ``` -Refactor `UserManager` to use `UserService` and `NotificationService` +refactor(user): replace UserDAO with service-based architecture -Replaced `UserDAO` with `UserService` and introduced `NotificationService` to improve separation of concerns and make user management logic reusable and extendable. +Introduce UserService and NotificationService to improve separation of +concerns and make user management logic more reusable. ``` **Input (diff)**: @@ -61,39 +99,142 @@ Replaced `UserDAO` with `UserService` and introduced `NotificationService` to im + accessAllowed = age > 17; ``` -**Output (commit message)**: +**Output**: ``` -Simplify age check logic for accessing permissions by using a single expression +refactor: simplify age check with ternary expression ``` -# Notes -- Make sure the commit messages are descriptive enough to convey why the change is being made without being too verbose. -- If applicable, add `Fixes #` or other references to link the commit to specific tickets. -- Avoid wording: "Updated", "Modified", or "Changed" without explicitly stating *why*—focus on *intent*."#; +**Input (diff)**: +```diff ++ export async function fetchUserProfile(userId: string) { ++ const response = await api.get(`/users/${userId}`); ++ return response.data; ++ } +``` -const DIFF_WITH_USERS_TEXT_PROMPT: &str = r#"Generate a commit message using the diff and the provided initial commit message as a template for context. +**Output**: +``` +feat(api): add user profile fetch endpoint +``` -[Additional details as needed.] +**Input (diff)**: +```diff +- const timeout = 5000; ++ const timeout = 30000; +``` -# Steps +**Output**: +``` +fix(database): increase query timeout to prevent failures -1. Analyze the code diff to understand the changes made. -2. Review the user's initial commit message to understand the intent and use it as a contextual starting point. -3. Determine the functionality added or removed, and the reason for these adjustments. -4. Combine insights from the diff and user's initial commit message to generate a more descriptive and complete commit message. -5. Summarize the details of the change in an accurate and informative, yet concise way. -6. Structure the message in a way that starts with a short summary line, followed by optional details if the change is complex. +Extend timeout from 5s to 30s to resolve query failures during peak load. -# Output Format +Fixes #234 +``` + +**Input (breaking change)**: +```diff +- function getUser(id) { return users[id]; } ++ function getUser(id) { return { user: users[id], metadata: {} }; } +``` + +**Output**: +``` +feat(api)!: wrap user response in object with metadata + +BREAKING CHANGE: getUser() now returns { user, metadata } instead of +user directly. Update all callers to access .user property. +``` + +# Important Guidelines + +- Choose the MOST significant type if changes span multiple categories +- Be specific in the description - avoid vague terms like "update", "fix stuff" +- The subject should complete: "If applied, this commit will " +- One commit = one logical change (if diff has unrelated changes, note it) +- Scope should reflect the module, component, or area affected"#; + +const DIFF_WITH_USERS_TEXT_PROMPT: &str = r#"Generate a commit message following Conventional Commits, using the user's input as context for intent. + +# Conventional Commits Format + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +## Commit Types (REQUIRED - choose exactly one) +- `feat`: New feature (correlates with MINOR in SemVer) +- `fix`: Bug fix (correlates with PATCH in SemVer) +- `refactor`: Code restructuring without changing behavior +- `perf`: Performance improvement +- `docs`: Documentation only changes +- `style`: Code style changes (formatting, whitespace, semicolons) +- `test`: Adding or correcting tests +- `build`: Changes to build system or dependencies +- `ci`: Changes to CI configuration +- `chore`: Maintenance tasks (tooling, configs, no production code change) +- `revert`: Reverting a previous commit + +## Rules + +### Subject Line (REQUIRED) +1. Format: `(): ` or `: ` +2. Use imperative mood ("add" not "added" or "adds") +3. Do NOT capitalize the first letter of description +4. Do NOT end with a period +5. Keep under 50 characters (hard limit: 72) +6. Scope is optional but recommended for larger projects + +### Body (OPTIONAL - use for complex changes) +1. Separate from subject with a blank line +2. Wrap at 72 characters +3. Explain WHAT and WHY, not HOW +4. Use bullet points for multiple items + +### Footer (OPTIONAL) +1. Reference issues: `Fixes #123`, `Closes #456`, `Refs #789` +2. Breaking changes: Start with `BREAKING CHANGE:` or add `!` after type +3. Co-authors: `Co-authored-by: Name ` + +## Breaking Changes +- Add `!` after type/scope: `feat!:` or `feat(api)!:` +- Or include `BREAKING CHANGE:` footer with explanation + +# Steps -The output should be a single commit message in the following format: -- A **first line summarizing** the purpose of the change. This line should be concise. -- Optionally, include a **second paragraph** with *additional context* if the change is complex or otherwise needs further clarification. - (e.g., if there's a bug fix, mention what problem was fixed and why the change works.) +1. Analyze the user's initial commit message to understand their intent +2. Analyze the diff to understand the actual changes +3. Determine the correct type based on the nature of changes +4. Extract or infer a scope from user input or affected files +5. Synthesize user intent + diff analysis into a proper conventional commit +6. If user mentions an issue number, include it in the footer # Examples -**Input (initial commit message)**: +**Input (user's message)**: +``` +fix the login bug +``` + +**Input (diff)**: +```diff +- if (user.password === input) { ++ if (await bcrypt.compare(input, user.passwordHash)) { +``` + +**Output**: +``` +fix(auth): use bcrypt for secure password comparison + +Replace plaintext password comparison with bcrypt hash verification +to fix authentication vulnerability. +``` + +**Input (user's message)**: ``` Refactor UserManager to use services instead of DAOs ``` @@ -102,49 +243,85 @@ Refactor UserManager to use services instead of DAOs ```diff - public class UserManager { - private final UserDAO userDAO; - + public class UserManager { + private final UserService userService; + private final NotificationService notificationService; +``` + +**Output**: +``` +refactor(user): replace UserDAO with service-based architecture + +Introduce UserService and NotificationService to improve separation of +concerns and make user management logic more reusable. +``` + +**Input (user's message)**: +``` +added new endpoint for users #123 +``` - public UserManager(UserDAO userDAO) { -- this.userDAO = userDAO; -+ this.userService = new UserService(); -+ this.notificationService = new NotificationService(); - } +**Input (diff)**: +```diff ++ @GetMapping("/users/{id}/preferences") ++ public ResponseEntity getUserPreferences(@PathVariable Long id) { ++ return ResponseEntity.ok(userService.getPreferences(id)); ++ } ``` -**Output (commit message)**: +**Output**: ``` -Refactor `UserManager` to use `UserService` and `NotificationService` +feat(api): add user preferences endpoint -Replaced `UserDAO` with `UserService` and introduced `NotificationService` to improve separation of concerns and make user management logic reusable and extendable. +Refs #123 ``` -**Input (initial commit message)**: +**Input (user's message)**: ``` -Simplify age check logic +cleanup ``` **Input (diff)**: ```diff -- if (age > 17) { -- accessAllowed = true; -- } else { -- accessAllowed = false; -- } -+ accessAllowed = age > 17; +- // TODO: implement later +- // console.log("debug"); +- const unusedVar = 42; +``` + +**Output**: +``` +chore: remove dead code and debug artifacts +``` + +**Input (user's message)**: +``` +BREAKING: change API response format +``` + +**Input (diff)**: +```diff +- return user; ++ return { data: user, version: "2.0" }; ``` -**Output (commit message)**: +**Output**: ``` -Simplify age check logic for accessing permissions by using a single expression +feat(api)!: wrap responses in versioned data envelope + +BREAKING CHANGE: All API responses now return { data, version } object +instead of raw data. Clients must access response.data for the payload. ``` -# Notes -- Make sure the commit messages are descriptive enough to convey why the change is being made without being too verbose. -- If applicable, add `Fixes #` or other references to link the commit to specific tickets. -- Avoid wording: "Updated", "Modified", or "Changed" without explicitly stating *why*—focus on *intent*."#; +# Important Guidelines + +- Preserve the user's intent but format it correctly +- If user mentions "bug", "fix", "broken" → likely `fix` +- If user mentions "add", "new", "feature" → likely `feat` +- If user mentions "refactor", "restructure", "reorganize" → `refactor` +- If user mentions "clean", "remove unused" → likely `chore` or `refactor` +- Extract issue numbers (#123) from user text and move to footer +- The subject should complete: "If applied, this commit will " +- Don't just paraphrase the user - analyze the diff to add specificity"#; const N_CTX: usize = 32000; const TEMPERATURE: f32 = 0.5; diff --git a/refact-agent/engine/src/agentic/mod.rs b/refact-agent/engine/src/agentic/mod.rs index 6a05dbfa3..6c3f144b9 100644 --- a/refact-agent/engine/src/agentic/mod.rs +++ b/refact-agent/engine/src/agentic/mod.rs @@ -1,3 +1,4 @@ pub mod generate_commit_message; pub mod generate_follow_up_message; -pub mod compress_trajectory; \ No newline at end of file +pub mod compress_trajectory; +pub mod generate_code_edit; \ No newline at end of file diff --git a/refact-agent/engine/src/at_commands/at_knowledge.rs b/refact-agent/engine/src/at_commands/at_knowledge.rs index 551872c09..f7001f844 100644 --- a/refact-agent/engine/src/at_commands/at_knowledge.rs +++ b/refact-agent/engine/src/at_commands/at_knowledge.rs @@ -9,16 +9,13 @@ use crate::at_commands::execute_at::AtCommandMember; use crate::call_validation::{ChatMessage, ContextEnum}; use crate::memories::memories_search; -/// @knowledge-load command - loads knowledge entries by search key or memory ID pub struct AtLoadKnowledge { params: Vec>, } impl AtLoadKnowledge { pub fn new() -> Self { - AtLoadKnowledge { - params: vec![], - } + AtLoadKnowledge { params: vec![] } } } @@ -38,31 +35,37 @@ impl AtCommand for AtLoadKnowledge { return Err("Usage: @knowledge-load ".to_string()); } - let search_key = args.iter().map(|x| x.text.clone()).join(" ").to_string(); - let gcx = { - let ccx_locked = ccx.lock().await; - ccx_locked.global_context.clone() - }; + let search_key = args.iter().map(|x| x.text.clone()).join(" "); + let gcx = ccx.lock().await.global_context.clone(); - let mem_top_n = 5; - let memories = memories_search(gcx.clone(), &search_key, mem_top_n).await?; + let memories = memories_search(gcx, &search_key, 5).await?; let mut seen_memids = HashSet::new(); let unique_memories: Vec<_> = memories.into_iter() - .filter(|m| seen_memids.insert(m.iknow_id.clone())) + .filter(|m| seen_memids.insert(m.memid.clone())) .collect(); - let mut results = String::new(); - for memory in unique_memories { - let mut content = String::new(); - content.push_str(&format!("🗃️{}\n", memory.iknow_id)); - content.push_str(&memory.iknow_memory); - results.push_str(&content); - }; + + let results = unique_memories.iter().map(|m| { + let mut result = String::new(); + if let Some(path) = &m.file_path { + result.push_str(&format!("📄 {}", path.display())); + if let Some((start, end)) = m.line_range { + result.push_str(&format!(":{}-{}", start, end)); + } + result.push('\n'); + } + if let Some(title) = &m.title { + result.push_str(&format!("📌 {}\n", title)); + } + result.push_str(&m.content); + result.push_str("\n\n"); + result + }).collect::(); let context = ContextEnum::ChatMessage(ChatMessage::new("plain_text".to_string(), results)); Ok((vec![context], "".to_string())) } fn depends_on(&self) -> Vec { - vec!["knowledge".to_string()] + vec![] } } diff --git a/refact-agent/engine/src/background_tasks.rs b/refact-agent/engine/src/background_tasks.rs index 4e871b3ec..a537614e7 100644 --- a/refact-agent/engine/src/background_tasks.rs +++ b/refact-agent/engine/src/background_tasks.rs @@ -39,16 +39,15 @@ impl BackgroundTasksHolder { } } -pub async fn start_background_tasks(gcx: Arc>, config_dir: &PathBuf) -> BackgroundTasksHolder { +pub async fn start_background_tasks(gcx: Arc>, _config_dir: &PathBuf) -> BackgroundTasksHolder { let mut bg = BackgroundTasksHolder::new(vec![ tokio::spawn(crate::files_in_workspace::files_in_workspace_init_task(gcx.clone())), tokio::spawn(crate::telemetry::basic_transmit::telemetry_background_task(gcx.clone())), tokio::spawn(crate::snippets_transmit::tele_snip_background_task(gcx.clone())), - tokio::spawn(crate::vecdb::vdb_highlev::vecdb_background_reload(gcx.clone())), // this in turn can create global_context::vec_db + tokio::spawn(crate::vecdb::vdb_highlev::vecdb_background_reload(gcx.clone())), tokio::spawn(crate::integrations::sessions::remove_expired_sessions_background_task(gcx.clone())), - tokio::spawn(crate::memories::memories_migration(gcx.clone(), config_dir.clone())), tokio::spawn(crate::git::cleanup::git_shadow_cleanup_background_task(gcx.clone())), - tokio::spawn(crate::cloud::threads_sub::watch_threads_subscription(gcx.clone())), + tokio::spawn(crate::knowledge_graph::knowledge_cleanup_background_task(gcx.clone())), ]); let ast = gcx.clone().read().await.ast_service.clone(); if let Some(ast_service) = ast { diff --git a/refact-agent/engine/src/cloud/experts_req.rs b/refact-agent/engine/src/cloud/experts_req.rs deleted file mode 100644 index 74fb9ac8b..000000000 --- a/refact-agent/engine/src/cloud/experts_req.rs +++ /dev/null @@ -1,130 +0,0 @@ -use log::error; -use regex::Regex; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use tokio::sync::RwLock as ARwLock; - -use crate::global_context::GlobalContext; - -#[derive(Debug, Serialize, Deserialize)] -pub struct Expert { - pub owner_fuser_id: Option, - pub owner_shared: bool, - pub located_fgroup_id: Option, - pub fexp_name: String, - pub fexp_system_prompt: String, - pub fexp_python_kernel: String, - pub fexp_block_tools: String, - pub fexp_allow_tools: String, -} - -impl Expert { - pub fn is_tool_allowed(&self, tool_name: &str) -> bool { - let mut blocked = false; - if !self.fexp_block_tools.trim().is_empty() { - match Regex::new(&self.fexp_block_tools) { - Ok(re) => { - if re.is_match(tool_name) { - blocked = true; - } - } - Err(e) => { - error!( - "Failed to compile fexp_block_tools regex: {}: {}", - self.fexp_block_tools, e - ); - } - } - } - // Allow if matches allow regex, even if blocked - if !self.fexp_allow_tools.trim().is_empty() { - match Regex::new(&self.fexp_allow_tools) { - Ok(re) => { - if re.is_match(tool_name) { - return true; - } - } - Err(e) => { - error!( - "Failed to compile fexp_allow_tools regex: {}: {}", - self.fexp_allow_tools, e - ); - } - } - } - - !blocked - } -} - -pub async fn get_expert( - gcx: Arc>, - expert_name: &str -) -> Result { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetExpert($id: String!) { - expert_get(id: $id) { - owner_fuser_id - owner_shared - located_fgroup_id - fexp_name - fexp_system_prompt - fexp_python_kernel - fexp_block_tools - fexp_allow_tools - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": { - "id": expert_name - } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(expert_value) = data.get("expert_get") { - let expert: Expert = serde_json::from_value(expert_value.clone()) - .map_err(|e| format!("Failed to parse expert: {}", e))?; - return Ok(expert); - } - } - Err(format!( - "Expert with name '{}' not found or unexpected response format: {}", - expert_name, response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get expert with name {}: HTTP status {}, error: {}", - expert_name, status, error_text - )) - } -} diff --git a/refact-agent/engine/src/cloud/messages_req.rs b/refact-agent/engine/src/cloud/messages_req.rs deleted file mode 100644 index e1c6a9bb0..000000000 --- a/refact-agent/engine/src/cloud/messages_req.rs +++ /dev/null @@ -1,365 +0,0 @@ -use crate::call_validation::{ChatContent, ChatMessage, ChatToolCall, ChatUsage, DiffChunk}; -use crate::global_context::GlobalContext; -use log::error; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use itertools::Itertools; -use tokio::sync::RwLock as ARwLock; -use tracing::warn; - -#[derive(Debug, Serialize, Deserialize)] -pub struct ThreadMessage { - pub ftm_belongs_to_ft_id: String, - pub ftm_alt: i32, - pub ftm_num: i32, - pub ftm_prev_alt: i32, - pub ftm_role: String, - pub ftm_content: Option, - pub ftm_tool_calls: Option, - pub ftm_call_id: String, - pub ftm_usage: Option, - pub ftm_created_ts: f64, - pub ftm_provenance: Value, -} - -pub async fn get_thread_messages( - gcx: Arc>, - thread_id: &str, - alt: i64, -) -> Result, String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetThreadMessagesByAlt($thread_id: String!, $alt: Int!) { - thread_messages_list( - ft_id: $thread_id, - ftm_alt: $alt - ) { - ftm_belongs_to_ft_id - ftm_alt - ftm_num - ftm_prev_alt - ftm_role - ftm_content - ftm_tool_calls - ftm_call_id - ftm_usage - ftm_created_ts - ftm_provenance - } - } - "#; - let variables = json!({ - "thread_id": thread_id, - "alt": alt - }); - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": variables - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(messages) = data.get("thread_messages_list") { - let messages: Vec = serde_json::from_value(messages.clone()) - .map_err(|e| format!("Failed to parse thread messages: {}", e))?; - return Ok(messages); - } - } - Err(format!("Unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread messages: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn create_thread_messages( - gcx: Arc>, - thread_id: &str, - messages: Vec, -) -> Result<(), String> { - if messages.is_empty() { - return Err("No messages provided".to_string()); - } - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let mut input_messages = Vec::with_capacity(messages.len()); - for message in messages { - if message.ftm_belongs_to_ft_id != thread_id { - return Err(format!( - "Message thread ID {} doesn't match the provided thread ID {}", - message.ftm_belongs_to_ft_id, thread_id - )); - } - if message.ftm_role.is_empty() { - return Err("Message role is required".to_string()); - } - let tool_calls_str = match &message.ftm_tool_calls { - Some(tc) => serde_json::to_string(tc) - .map_err(|e| format!("Failed to serialize tool_calls: {}", e))?, - None => "{}".to_string(), - }; - let usage_str = match &message.ftm_usage { - Some(u) => { - serde_json::to_string(u).map_err(|e| format!("Failed to serialize usage: {}", e))? - } - None => "{}".to_string(), - }; - input_messages.push(json!({ - "ftm_belongs_to_ft_id": message.ftm_belongs_to_ft_id, - "ftm_alt": message.ftm_alt, - "ftm_num": message.ftm_num, - "ftm_prev_alt": message.ftm_prev_alt, - "ftm_role": message.ftm_role, - "ftm_content": serde_json::to_string(&message.ftm_content).unwrap(), - "ftm_tool_calls": tool_calls_str, - "ftm_call_id": message.ftm_call_id, - "ftm_usage": usage_str, - "ftm_provenance": serde_json::to_string(&message.ftm_provenance).unwrap() - })); - } - let variables = json!({ - "input": { - "ftm_belongs_to_ft_id": thread_id, - "messages": input_messages - } - }); - let mutation = r#" - mutation ThreadMessagesCreateMultiple($input: FThreadMultipleMessagesInput!) { - thread_messages_create_multiple(input: $input) { - count - messages { - ftm_belongs_to_ft_id - ftm_alt - ftm_num - ftm_prev_alt - ftm_role - ftm_created_ts - ftm_call_id - ftm_provenance - } - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": mutation, - "variables": variables - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(_) = response_json.get("data") { - return Ok(()) - } - Err(format!("Unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to create thread messages: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub fn convert_thread_messages_to_messages( - thread_messages: &Vec, -) -> Vec { - thread_messages - .iter() - .map(|msg| { - let content: ChatContent = if let Some(content) = &msg.ftm_content { - serde_json::from_value(content.clone()).unwrap_or_default() - } else { - ChatContent::default() - }; - let tool_calls = msg.ftm_tool_calls.clone().map(|tc| { - serde_json::from_value::>(tc).unwrap_or_else(|_| vec![]) - }); - ChatMessage { - role: msg.ftm_role.clone(), - content, - tool_calls, - tool_call_id: msg.ftm_call_id.clone(), - tool_failed: None, - usage: msg.ftm_usage.clone().map(|u| { - serde_json::from_value::(u).unwrap_or_else(|_| ChatUsage::default()) - }), - checkpoints: vec![], - thinking_blocks: None, - finish_reason: None, - output_filter: None, - } - }) - .collect() -} - -pub fn convert_messages_to_thread_messages( - messages: Vec, - alt: i32, - prev_alt: i32, - start_num: i32, - thread_id: &str, -) -> Result, String> { - let mut output_messages = Vec::new(); - let flush_delayed_images = |results: &mut Vec, delay_images: &mut Vec| { - results.extend(delay_images.clone()); - delay_images.clear(); - }; - for (i, msg) in messages.into_iter().enumerate() { - let num = start_num + i as i32; - let mut delay_images = vec![]; - let mut messages = if msg.role == "tool" { - let mut results = vec![]; - match &msg.content { - ChatContent::Multimodal(multimodal_content) => { - let texts = multimodal_content.iter().filter(|x|x.is_text()).collect::>(); - let images = multimodal_content.iter().filter(|x|x.is_image()).collect::>(); - let text = if texts.is_empty() { - "attached images below".to_string() - } else { - texts.iter().map(|x|x.m_content.clone()).collect::>().join("\n") - }; - let mut msg_cloned = msg.clone(); - msg_cloned.content = ChatContent::SimpleText(text); - results.push(msg_cloned.into_value(&None, "")); - if !images.is_empty() { - let msg_img = ChatMessage { - role: "user".to_string(), - content: ChatContent::Multimodal(images.into_iter().cloned().collect()), - ..Default::default() - }; - delay_images.push(msg_img.into_value(&None, "")); - } - }, - ChatContent::SimpleText(_) => { - results.push(msg.into_value(&None, "")); - } - } - results - } else if msg.role == "assistant" || msg.role == "system" { - vec![msg.into_value(&None, "")] - } else if msg.role == "user" { - vec![msg.into_value(&None, "")] - } else if msg.role == "diff" { - let extra_message = match serde_json::from_str::>(&msg.content.content_text_only()) { - Ok(chunks) => { - if chunks.is_empty() { - "Nothing has changed.".to_string() - } else { - chunks.iter() - .filter(|x| !x.application_details.is_empty()) - .map(|x| x.application_details.clone()) - .join("\n") - } - }, - Err(_) => "".to_string() - }; - vec![ChatMessage { - role: "diff".to_string(), - content: ChatContent::SimpleText(format!("The operation has succeeded.\n{extra_message}")), - tool_calls: None, - tool_call_id: msg.tool_call_id.clone(), - ..Default::default() - }.into_value(&None, "")] - } else if msg.role == "plain_text" || msg.role == "cd_instruction" || msg.role == "context_file" { - vec![ChatMessage::new( - msg.role.to_string(), - msg.content.content_text_only(), - ).into_value(&None, "")] - } else { - warn!("unknown role: {}", msg.role); - vec![] - }; - flush_delayed_images(&mut messages, &mut delay_images); - for pp_msg in messages { - let tool_calls = pp_msg.get("tool_calls") - .map(|x| Some(x.clone())).unwrap_or(None); - let usage = pp_msg.get("usage") - .map(|x| Some(x.clone())).unwrap_or(None); - let content = pp_msg - .get("content") - .cloned() - .ok_or("cannot find role in the message".to_string())?; - output_messages.push(ThreadMessage { - ftm_belongs_to_ft_id: thread_id.to_string(), - ftm_alt: alt, - ftm_num: num, - ftm_prev_alt: prev_alt, - ftm_role: msg.role.clone(), - ftm_content: Some(content), - ftm_tool_calls: tool_calls, - ftm_call_id: msg.tool_call_id.clone(), - ftm_usage: usage, - ftm_created_ts: std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_secs_f64(), - ftm_provenance: json!({"system_type": "refact_lsp", "version": env!("CARGO_PKG_VERSION") }), - }) - } - } - Ok(output_messages) -} - -pub async fn get_tool_names_from_openai_format( - toolset_json: &Vec, -) -> Result, String> { - let mut tool_names = Vec::new(); - for tool in toolset_json { - if let Some(function) = tool.get("function") { - if let Some(name) = function.get("name") { - if let Some(name_str) = name.as_str() { - tool_names.push(name_str.to_string()); - } - } - } - } - Ok(tool_names) -} diff --git a/refact-agent/engine/src/cloud/mod.rs b/refact-agent/engine/src/cloud/mod.rs deleted file mode 100644 index e24f2d519..000000000 --- a/refact-agent/engine/src/cloud/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod threads_sub; -mod threads_req; -mod messages_req; -mod experts_req; diff --git a/refact-agent/engine/src/cloud/threads_req.rs b/refact-agent/engine/src/cloud/threads_req.rs deleted file mode 100644 index fbd74d4b9..000000000 --- a/refact-agent/engine/src/cloud/threads_req.rs +++ /dev/null @@ -1,300 +0,0 @@ -use log::error; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use tokio::sync::RwLock as ARwLock; - -use crate::global_context::GlobalContext; - -#[derive(Debug, Serialize, Deserialize)] -pub struct Thread { - pub owner_fuser_id: String, - pub owner_shared: bool, - pub located_fgroup_id: String, - pub ft_id: String, - pub ft_fexp_name: String, - pub ft_title: String, - pub ft_toolset: Vec, - pub ft_belongs_to_fce_id: Option, - pub ft_model: String, - pub ft_temperature: f64, - pub ft_max_new_tokens: i32, - pub ft_n: i32, - pub ft_error: Option, - pub ft_need_assistant: i32, - pub ft_need_tool_calls: i32, - pub ft_anything_new: bool, - pub ft_created_ts: f64, - pub ft_updated_ts: f64, - pub ft_archived_ts: f64, - pub ft_locked_by: String, -} - -pub async fn get_thread( - gcx: Arc>, - thread_id: &str, -) -> Result { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetThread($id: String!) { - thread_get(id: $id) { - owner_fuser_id - owner_shared - located_fgroup_id - ft_id - ft_fexp_name, - ft_title - ft_belongs_to_fce_id - ft_model - ft_temperature - ft_max_new_tokens - ft_n - ft_error - ft_toolset - ft_need_assistant - ft_need_tool_calls - ft_anything_new - ft_created_ts - ft_updated_ts - ft_archived_ts - ft_locked_by - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": {"id": thread_id} - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(thread_value) = data.get("thread_get") { - let thread: Thread = serde_json::from_value(thread_value.clone()) - .map_err(|e| format!("Failed to parse thread: {}", e))?; - return Ok(thread); - } - } - Err(format!( - "Thread not found or unexpected response format: {}", - response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn set_thread_toolset( - gcx: Arc>, - thread_id: &str, - ft_toolset: Vec, -) -> Result, String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let mutation = r#" - mutation UpdateThread($thread_id: String!, $patch: FThreadPatch!) { - thread_patch(id: $thread_id, patch: $patch) { - ft_toolset - } - } - "#; - let variables = json!({ - "thread_id": thread_id, - "patch": { - "ft_toolset": serde_json::to_string(&ft_toolset).unwrap() - } - }); - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": mutation, - "variables": variables - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(ft_toolset_json) = data.get("thread_patch") { - let ft_toolset: Vec = - serde_json::from_value(ft_toolset_json["ft_toolset"].clone()) - .map_err(|e| format!("Failed to parse updated thread: {}", e))?; - return Ok(ft_toolset); - } - } - Err(format!("Unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to update thread: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn lock_thread( - gcx: Arc>, - thread_id: &str, - hash: &str, -) -> Result<(), String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let worker_name = format!("refact-lsp:{hash}"); - let query = r#" - mutation AdvanceLock($ft_id: String!, $worker_name: String!) { - thread_lock(ft_id: $ft_id, worker_name: $worker_name) - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": {"ft_id": thread_id, "worker_name": worker_name} - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if data.get("thread_lock").is_some() { - return Ok(()); - } else { - return Err(format!("Thread {thread_id} is locked by another worker")); - } - } - Err(format!( - "Thread not found or unexpected response format: {}", - response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn unlock_thread( - gcx: Arc>, - thread_id: String, - hash: String, -) -> Result<(), String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let worker_name = format!("refact-lsp:{hash}"); - let query = r#" - mutation AdvanceUnlock($ft_id: String!, $worker_name: String!) { - thread_unlock(ft_id: $ft_id, worker_name: $worker_name) - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": {"ft_id": thread_id, "worker_name": worker_name} - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if data.get("thread_unlock").is_some() { - return Ok(()); - } else { - return Err(format!("Cannot unlock thread {thread_id}")); - } - } - Err(format!( - "Thread not found or unexpected response format: {}", - response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread: HTTP status {}, error: {}", - status, error_text - )) - } -} diff --git a/refact-agent/engine/src/cloud/threads_sub.rs b/refact-agent/engine/src/cloud/threads_sub.rs deleted file mode 100644 index f15998e48..000000000 --- a/refact-agent/engine/src/cloud/threads_sub.rs +++ /dev/null @@ -1,512 +0,0 @@ -use std::collections::HashSet; -use crate::global_context::GlobalContext; -use futures::{SinkExt, StreamExt}; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use std::sync::atomic::Ordering; -use std::time::Duration; -use indexmap::IndexMap; -use tokio::sync::RwLock as ARwLock; -use tokio::sync::Mutex as AMutex; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::{connect_async, tungstenite::protocol::Message}; -use tracing::{error, info, warn}; -use url::Url; -use crate::at_commands::at_commands::AtCommandsContext; -use crate::call_validation::ChatContent; -use crate::cloud::messages_req::ThreadMessage; -use crate::cloud::threads_req::{lock_thread, Thread}; -use rand::{Rng, thread_rng}; -use rand::distributions::Alphanumeric; -use crate::custom_error::MapErrToString; - - -const RECONNECT_DELAY_SECONDS: u64 = 3; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ThreadPayload { - pub owner_fuser_id: String, - pub ft_id: String, - pub ft_error: Option, - pub ft_locked_by: String, - pub ft_need_tool_calls: i64, - pub ft_app_searchable: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct BasicStuff { - pub fuser_id: String, - pub workspaces: Vec, -} - -const THREADS_SUBSCRIPTION_QUERY: &str = r#" - subscription ThreadsPageSubs($located_fgroup_id: String!) { - threads_in_group(located_fgroup_id: $located_fgroup_id) { - news_action - news_payload_id - news_payload { - owner_fuser_id - ft_id - ft_error - ft_locked_by - ft_need_tool_calls - ft_app_searchable - } - } - } -"#; - -pub async fn trigger_threads_subscription_restart(gcx: Arc>) { - let restart_flag = gcx.read().await.threads_subscription_restart_flag.clone(); - restart_flag.store(true, Ordering::SeqCst); - info!("threads subscription restart triggered"); -} - -pub async fn watch_threads_subscription(gcx: Arc>) { - if !gcx.read().await.cmdline.cloud_threads { - return; - } - - loop { - { - let restart_flag = gcx.read().await.threads_subscription_restart_flag.clone(); - restart_flag.store(false, Ordering::SeqCst); - } - let located_fgroup_id = if let Some(located_fgroup_id) = gcx.read().await.active_group_id.clone() { - located_fgroup_id - } else { - warn!("no active group is set, skipping threads subscription"); - tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - continue; - }; - - info!( - "starting subscription for threads_in_group with fgroup_id=\"{}\"", - located_fgroup_id - ); - let connection_result = initialize_connection(gcx.clone()).await; - let mut connection = match connection_result { - Ok(conn) => conn, - Err(err) => { - error!("failed to initialize connection: {}", err); - info!("will attempt to reconnect in {} seconds", RECONNECT_DELAY_SECONDS); - tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - continue; - } - }; - - let events_result = events_loop(gcx.clone(), &mut connection).await; - if let Err(err) = events_result { - error!("failed to process events: {}", err); - info!("will attempt to reconnect in {} seconds", RECONNECT_DELAY_SECONDS); - } - - if gcx.read().await.shutdown_flag.load(Ordering::SeqCst) { - info!("shutting down threads subscription"); - break; - } - - let restart_flag = gcx.read().await.threads_subscription_restart_flag.clone(); - if !restart_flag.load(Ordering::SeqCst) { - tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - } - } -} - -async fn initialize_connection(gcx: Arc>) -> Result< - futures::stream::SplitStream< - tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream - >, - >, - String, -> { - let (api_key, located_fgroup_id) = { - let gcx_read = gcx.read().await; - (gcx_read.cmdline.api_key.clone(), - gcx_read.active_group_id.clone().unwrap_or_default()) - }; - let url = Url::parse(crate::constants::GRAPHQL_WS_URL) - .map_err(|e| format!("Failed to parse WebSocket URL: {}", e))?; - let mut request = url - .into_client_request() - .map_err(|e| format!("Failed to create request: {}", e))?; - request - .headers_mut() - .insert("Sec-WebSocket-Protocol", "graphql-ws".parse().unwrap()); - let (ws_stream, _) = connect_async(request) - .await - .map_err(|e| format!("Failed to connect to WebSocket server: {}", e))?; - let (mut write, mut read) = ws_stream.split(); - let init_message = json!({ - "type": "connection_init", - "payload": { - "apikey": api_key - } - }); - write.send(Message::Text(init_message.to_string())).await - .map_err(|e| format!("Failed to send connection init message: {}", e))?; - - let timeout = tokio::time::timeout(Duration::from_secs(5), read.next()) - .await - .map_err(|_| "Timeout waiting for connection acknowledgment".to_string())?; - - if let Some(msg) = timeout { - let msg = msg.map_err(|e| format!("WebSocket error: {}", e))?; - match msg { - Message::Text(text) => { - info!("Received response: {}", text); - let response: Value = serde_json::from_str(&text) - .map_err(|e| format!("Failed to parse connection response: {}", e))?; - if let Some(msg_type) = response["type"].as_str() { - if msg_type == "connection_ack" { - } else if msg_type == "connection_error" { - return Err(format!("Connection error: {}", response)); - } else { - return Err(format!("Expected connection_ack, got: {}", response)); - } - } else { - return Err(format!( - "Invalid response format, missing 'type': {}", - response - )); - } - } - Message::Close(frame) => { - return if let Some(frame) = frame { - Err(format!( - "WebSocket closed during initialization: code={}, reason={}", - frame.code, frame.reason - )) - } else { - Err( - "WebSocket connection closed during initialization without details" - .to_string(), - ) - } - } - _ => { - return Err(format!("Unexpected message type received: {:?}", msg)); - } - } - } else { - return Err("No response received for connection initialization".to_string()); - } - let subscription_message = json!({ - "id": "42", - "type": "start", - "payload": { - "query": THREADS_SUBSCRIPTION_QUERY, - "variables": { - "located_fgroup_id": located_fgroup_id - } - } - }); - write - .send(Message::Text(subscription_message.to_string())) - .await - .map_err(|e| format!("Failed to send subscription message: {}", e))?; - Ok(read) -} - -async fn events_loop( - gcx: Arc>, - connection: &mut futures::stream::SplitStream< - tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, - >, -) -> Result<(), String> { - info!("cloud threads subscription started, waiting for events..."); - let basic_info = get_basic_info(gcx.clone()).await?; - while let Some(msg) = connection.next().await { - if gcx.read().await.shutdown_flag.load(Ordering::SeqCst) { - info!("shutting down threads subscription"); - break; - } - if gcx.read().await.threads_subscription_restart_flag.load(Ordering::SeqCst) { - info!("restart flag detected, restarting threads subscription"); - return Ok(()); - } - match msg { - Ok(Message::Text(text)) => { - let response: Value = match serde_json::from_str(&text) { - Ok(res) => res, - Err(err) => { - error!("failed to parse message: {}, error: {}", text, err); - continue; - } - }; - let response_type = response["type"].as_str().unwrap_or("unknown"); - match response_type { - "data" => { - if let Some(payload) = response["payload"].as_object() { - let data = &payload["data"]; - let threads_in_group = &data["threads_in_group"]; - let news_action = threads_in_group["news_action"].as_str().unwrap_or(""); - if news_action != "INSERT" && news_action != "UPDATE" { - continue; - } - if let Ok(payload) = serde_json::from_value::(threads_in_group["news_payload"].clone()) { - match process_thread_event(gcx.clone(), &payload, &basic_info).await { - Ok(_) => {} - Err(err) => { - error!("failed to process thread event: {}", err); - } - } - } else { - info!("failed to parse thread payload: {}", text); - } - } else { - info!("received data message but couldn't find payload"); - } - } - "error" => { - error!("threads subscription error: {}", text); - } - _ => { - info!("received message with unknown type: {}", text); - } - } - } - Ok(Message::Close(_)) => { - info!("webSocket connection closed"); - break; - } - Ok(_) => {} - Err(e) => { - return Err(format!("webSocket error: {}", e)); - } - } - } - Ok(()) -} -fn generate_random_hash(length: usize) -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(length) - .map(char::from) - .collect() -} - -async fn get_basic_info(gcx: Arc>) -> Result { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetBasicInfo { - query_basic_stuff { - fuser_id - workspaces { - ws_id - ws_owner_fuser_id - ws_root_group_id - root_group_name - } - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({"query": query})) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - return Err(format!("GraphQL request error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - let basic_stuff_struct: BasicStuff = serde_json::from_value(data["query_basic_stuff"].clone()) - .map_err(|e| format!("Failed to parse updated thread: {}", e))?; - return Ok(basic_stuff_struct); - } - Err(format!("Basic data not found or unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get basic data: HTTP status {}, error: {}", - status, error_text - )) - } -} - -async fn process_thread_event( - gcx: Arc>, - thread_payload: &ThreadPayload, - basic_info: &BasicStuff -) -> Result<(), String> { - if thread_payload.ft_need_tool_calls == -1 || thread_payload.owner_fuser_id != basic_info.fuser_id { - return Ok(()); - } - let app_searchable_id = gcx.read().await.app_searchable_id.clone(); - if let Some(ft_app_searchable) = thread_payload.ft_app_searchable.clone() { - if ft_app_searchable != app_searchable_id { - info!("thread `{}` has different `app_searchable` id, skipping it", thread_payload.ft_id); - } - } else { - info!("thread `{}` doesn't have the `app_searchable` id, skipping it", thread_payload.ft_id); - return Ok(()); - } - if let Some(error) = thread_payload.ft_error.as_ref() { - info!("thread `{}` has the error: `{}`. Skipping it", thread_payload.ft_id, error); - return Ok(()); - } - let messages = crate::cloud::messages_req::get_thread_messages( - gcx.clone(), - &thread_payload.ft_id, - thread_payload.ft_need_tool_calls, - ).await?; - if messages.is_empty() { - info!("thread `{}` has no messages. Skipping it", thread_payload.ft_id); - return Ok(()); - } - let thread = crate::cloud::threads_req::get_thread(gcx.clone(), &thread_payload.ft_id).await?; - let hash = generate_random_hash(16); - match lock_thread(gcx.clone(), &thread.ft_id, &hash).await { - Ok(_) => {} - Err(err) => return Err(err) - } - let result = if messages.iter().all(|x| x.ftm_role != "system") { - initialize_thread(gcx.clone(), &thread.ft_fexp_name, &thread, &messages).await - } else { - call_tools(gcx.clone(), &thread, &messages).await - }; - match crate::cloud::threads_req::unlock_thread(gcx.clone(), thread.ft_id.clone(), hash).await { - Ok(_) => info!("thread `{}` unlocked successfully", thread.ft_id), - Err(err) => error!("failed to unlock thread `{}`: {}", thread.ft_id, err), - } - result -} - -async fn initialize_thread( - gcx: Arc>, - expert_name: &str, - thread: &Thread, - thread_messages: &Vec, -) -> Result<(), String> { - let expert = crate::cloud::experts_req::get_expert(gcx.clone(), expert_name).await?; - let tools: Vec> = - crate::tools::tools_list::get_available_tools(gcx.clone()) - .await - .into_iter() - .filter(|tool| expert.is_tool_allowed(&tool.tool_description().name)) - .collect(); - let tool_descriptions = tools - .iter() - .map(|x| x.tool_description().into_openai_style()) - .collect::>(); - crate::cloud::threads_req::set_thread_toolset(gcx.clone(), &thread.ft_id, tool_descriptions).await?; - let updated_system_prompt = crate::scratchpads::chat_utils_prompts::system_prompt_add_extra_instructions( - gcx.clone(), expert.fexp_system_prompt.clone(), HashSet::new(), &crate::call_validation::ChatMeta::default() - ).await; - let last_message = thread_messages.last().unwrap(); - let output_thread_messages = vec![ThreadMessage { - ftm_belongs_to_ft_id: last_message.ftm_belongs_to_ft_id.clone(), - ftm_alt: last_message.ftm_alt.clone(), - ftm_num: 0, - ftm_prev_alt: 100, - ftm_role: "system".to_string(), - ftm_content: Some( - serde_json::to_value(ChatContent::SimpleText(updated_system_prompt)).unwrap(), - ), - ftm_tool_calls: None, - ftm_call_id: "".to_string(), - ftm_usage: None, - ftm_created_ts: std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs_f64(), - ftm_provenance: json!({"important": "information"}), - }]; - crate::cloud::messages_req::create_thread_messages( - gcx.clone(), - &thread.ft_id, - output_thread_messages, - ).await?; - Ok(()) -} - -async fn call_tools( - gcx: Arc>, - thread: &Thread, - thread_messages: &Vec, -) -> Result<(), String> { - let max_new_tokens = 8192; - let last_message_num = thread_messages.iter().map(|x| x.ftm_num).max().unwrap_or(0); - let (alt, prev_alt) = thread_messages - .last() - .map(|msg| (msg.ftm_alt, msg.ftm_prev_alt)) - .unwrap_or((0, 0)); - let messages = crate::cloud::messages_req::convert_thread_messages_to_messages(thread_messages); - let caps = crate::global_context::try_load_caps_quickly_if_not_present(gcx.clone(), 0) - .await - .map_err_to_string()?; - let model_rec = crate::caps::resolve_chat_model(caps, &format!("refact/{}", thread.ft_model)) - .map_err(|e| format!("Failed to resolve chat model: {}", e))?; - let ccx = Arc::new(AMutex::new( - AtCommandsContext::new( - gcx.clone(), - model_rec.base.n_ctx, - 12, - false, - messages.clone(), - thread.ft_id.to_string(), - false, - thread.ft_model.to_string(), - ).await, - )); - let allowed_tools = crate::cloud::messages_req::get_tool_names_from_openai_format(&thread.ft_toolset).await?; - let mut all_tools: IndexMap> = - crate::tools::tools_list::get_available_tools(gcx.clone()).await - .into_iter() - .filter(|x| allowed_tools.contains(&x.tool_description().name)) - .map(|x| (x.tool_description().name, x)) - .collect(); - let mut has_rag_results = crate::scratchpads::scratchpad_utils::HasRagResults::new(); - let tokenizer_arc = crate::tokens::cached_tokenizer(gcx.clone(), &model_rec.base).await?; - let messages_count = messages.len(); - let (output_messages, _) = crate::tools::tools_execute::run_tools_locally( - ccx.clone(), - &mut all_tools, - tokenizer_arc, - max_new_tokens, - &messages, - &mut has_rag_results, - &None, - ).await?; - if messages.len() == output_messages.len() { - tracing::warn!( - "Thread has no active tool call awaiting but still has need_tool_call turned on" - ); - return Ok(()); - } - let output_thread_messages = crate::cloud::messages_req::convert_messages_to_thread_messages( - output_messages.into_iter().skip(messages_count).collect(), - alt, - prev_alt, - last_message_num + 1, - &thread.ft_id, - )?; - crate::cloud::messages_req::create_thread_messages( - gcx.clone(), - &thread.ft_id, - output_thread_messages, - ).await?; - Ok(()) -} diff --git a/refact-agent/engine/src/constants.rs b/refact-agent/engine/src/constants.rs index 248ad7bcf..aff197fc6 100644 --- a/refact-agent/engine/src/constants.rs +++ b/refact-agent/engine/src/constants.rs @@ -1,3 +1 @@ pub const CLOUD_URL: &str = "https://flexus.team/v1"; -pub const GRAPHQL_WS_URL: &str = "ws://flexus.team/v1/graphql"; -pub const GRAPHQL_URL: &str = "https://flexus.team/v1/graphql"; diff --git a/refact-agent/engine/src/file_filter.rs b/refact-agent/engine/src/file_filter.rs index 4b75075c8..46539bd07 100644 --- a/refact-agent/engine/src/file_filter.rs +++ b/refact-agent/engine/src/file_filter.rs @@ -6,6 +6,10 @@ use std::path::PathBuf; const LARGE_FILE_SIZE_THRESHOLD: u64 = 4096*1024; // 4Mb files const SMALL_FILE_SIZE_THRESHOLD: u64 = 5; // 5 Bytes +pub const KNOWLEDGE_FOLDER_NAME: &str = ".refact_knowledge"; + +const ALLOWED_HIDDEN_FOLDERS: &[&str] = &[KNOWLEDGE_FOLDER_NAME]; + pub const SOURCE_FILE_EXTENSIONS: &[&str] = &[ "c", "cpp", "cc", "h", "hpp", "cs", "java", "py", "rb", "go", "rs", "swift", "php", "js", "jsx", "ts", "tsx", "lua", "pl", "r", "sh", "bat", "cmd", "ps1", @@ -16,12 +20,22 @@ pub const SOURCE_FILE_EXTENSIONS: &[&str] = &[ "gradle", "liquid" ]; +fn is_in_allowed_hidden_folder(path: &PathBuf) -> bool { + path.ancestors().any(|ancestor| { + ancestor.file_name() + .map(|name| ALLOWED_HIDDEN_FOLDERS.contains(&name.to_string_lossy().as_ref())) + .unwrap_or(false) + }) +} + pub fn is_valid_file(path: &PathBuf, allow_hidden_folders: bool, ignore_size_thresholds: bool) -> Result<(), Box> { if !path.is_file() { return Err("Path is not a file".into()); } - if !allow_hidden_folders && path.ancestors().any(|ancestor| { + let in_allowed_hidden = is_in_allowed_hidden_folder(path); + + if !allow_hidden_folders && !in_allowed_hidden && path.ancestors().any(|ancestor| { ancestor.file_name() .map(|name| name.to_string_lossy().starts_with('.')) .unwrap_or(false) diff --git a/refact-agent/engine/src/files_in_workspace.rs b/refact-agent/engine/src/files_in_workspace.rs index 04752edc1..6c96fbc99 100644 --- a/refact-agent/engine/src/files_in_workspace.rs +++ b/refact-agent/engine/src/files_in_workspace.rs @@ -658,7 +658,6 @@ pub async fn on_workspaces_init(gcx: Arc>) -> i32 let new_app_searchable_id = get_app_searchable_id(&folders); if old_app_searchable_id != new_app_searchable_id { gcx.write().await.app_searchable_id = get_app_searchable_id(&folders); - crate::cloud::threads_sub::trigger_threads_subscription_restart(gcx.clone()).await; } watcher_init(gcx.clone()).await; let files_enqueued = enqueue_all_files_from_workspace_folders(gcx.clone(), false, false).await; diff --git a/refact-agent/engine/src/global_context.rs b/refact-agent/engine/src/global_context.rs index 4b6797640..c3acd17be 100644 --- a/refact-agent/engine/src/global_context.rs +++ b/refact-agent/engine/src/global_context.rs @@ -107,8 +107,6 @@ pub struct CommandLine { #[structopt(long, help="An pre-setup active group id")] pub active_group_id: Option, - #[structopt(long, help="Enable cloud threads support")] - pub cloud_threads: bool, } impl CommandLine { @@ -180,7 +178,6 @@ pub struct GlobalContext { pub init_shadow_repos_lock: Arc>, pub git_operations_abort_flag: Arc, pub app_searchable_id: String, - pub threads_subscription_restart_flag: Arc } pub type SharedGlobalContext = Arc>; // TODO: remove this type alias, confusing @@ -429,7 +426,6 @@ pub async fn create_global_context( init_shadow_repos_lock: Arc::new(AMutex::new(false)), git_operations_abort_flag: Arc::new(AtomicBool::new(false)), app_searchable_id: get_app_searchable_id(&workspace_dirs), - threads_subscription_restart_flag: Arc::new(AtomicBool::new(false)), }; let gcx = Arc::new(ARwLock::new(cx)); crate::files_in_workspace::watcher_init(gcx.clone()).await; diff --git a/refact-agent/engine/src/http/routers/v1.rs b/refact-agent/engine/src/http/routers/v1.rs index 8e3c86a8a..18ea86c60 100644 --- a/refact-agent/engine/src/http/routers/v1.rs +++ b/refact-agent/engine/src/http/routers/v1.rs @@ -35,8 +35,10 @@ use crate::http::routers::v1::providers::{handle_v1_providers, handle_v1_provide handle_v1_delete_model, handle_v1_delete_provider, handle_v1_model_default, handle_v1_completion_model_families}; use crate::http::routers::v1::vecdb::{handle_v1_vecdb_search, handle_v1_vecdb_status}; +use crate::http::routers::v1::knowledge_graph::handle_v1_knowledge_graph; use crate::http::routers::v1::v1_integrations::{handle_v1_integration_get, handle_v1_integration_icon, handle_v1_integration_save, handle_v1_integration_delete, handle_v1_integrations, handle_v1_integrations_filtered, handle_v1_integrations_mcp_logs}; use crate::http::routers::v1::file_edit_tools::handle_v1_file_edit_tool_dry_run; +use crate::http::routers::v1::code_edit::handle_v1_code_edit; use crate::http::routers::v1::workspace::{handle_v1_get_app_searchable_id, handle_v1_set_active_group_id}; mod ast; @@ -64,9 +66,11 @@ pub mod telemetry_chat; pub mod telemetry_network; pub mod providers; mod file_edit_tools; +mod code_edit; mod v1_integrations; pub mod vecdb; mod workspace; +mod knowledge_graph; pub fn make_v1_router() -> Router { let builder = Router::new() @@ -135,6 +139,7 @@ pub fn make_v1_router() -> Router { .route("/links", post(handle_v1_links)) .route("/file_edit_tool_dry_run", post(handle_v1_file_edit_tool_dry_run)) + .route("/code-edit", post(handle_v1_code_edit)) .route("/providers", get(handle_v1_providers)) .route("/provider-templates", get(handle_v1_provider_templates)) @@ -165,6 +170,7 @@ pub fn make_v1_router() -> Router { let builder = builder .route("/vdb-search", post(handle_v1_vecdb_search)) .route("/vdb-status", get(handle_v1_vecdb_status)) + .route("/knowledge-graph", get(handle_v1_knowledge_graph)) .route("/trajectory-save", post(handle_v1_trajectory_save)) .route("/trajectory-compress", post(handle_v1_trajectory_compress)) ; diff --git a/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs b/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs index fd20c83c9..4b2a91371 100644 --- a/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs +++ b/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs @@ -79,24 +79,18 @@ pub async fn handle_v1_trajectory_save( body_bytes: hyper::body::Bytes, ) -> axum::response::Result, ScratchError> { let post = serde_json::from_slice::(&body_bytes).map_err(|e| { - ScratchError::new( - StatusCode::UNPROCESSABLE_ENTITY, - format!("JSON problem: {}", e), - ) + ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, format!("JSON problem: {}", e)) })?; + let trajectory = compress_trajectory(gcx.clone(), &post.messages) .await.map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, e))?; - crate::memories::memories_add( - gcx.clone(), - "trajectory", - &trajectory.as_str(), - false, - ).await.map_err(|e| { - ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("{}", e)) - })?; + + let file_path = crate::memories::save_trajectory(gcx, &trajectory) + .await.map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, e))?; let response = serde_json::json!({ "trajectory": trajectory, + "file_path": file_path.to_string_lossy(), }); Ok(Response::builder() diff --git a/refact-agent/engine/src/http/routers/v1/code_edit.rs b/refact-agent/engine/src/http/routers/v1/code_edit.rs new file mode 100644 index 000000000..a4495ed71 --- /dev/null +++ b/refact-agent/engine/src/http/routers/v1/code_edit.rs @@ -0,0 +1,52 @@ +use crate::agentic::generate_code_edit::generate_code_edit; +use crate::custom_error::ScratchError; +use crate::global_context::GlobalContext; +use axum::http::{Response, StatusCode}; +use axum::Extension; +use hyper::Body; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::RwLock as ARwLock; + +#[derive(Deserialize)] +pub struct CodeEditPost { + pub code: String, + pub instruction: String, + pub cursor_file: String, + pub cursor_line: i32, +} + +#[derive(Serialize)] +pub struct CodeEditResponse { + pub edited_code: String, +} + +pub async fn handle_v1_code_edit( + Extension(global_context): Extension>>, + body_bytes: hyper::body::Bytes, +) -> axum::response::Result, ScratchError> { + let post = serde_json::from_slice::(&body_bytes).map_err(|e| { + ScratchError::new( + StatusCode::UNPROCESSABLE_ENTITY, + format!("JSON problem: {}", e), + ) + })?; + + let edited_code = generate_code_edit( + global_context.clone(), + &post.code, + &post.instruction, + &post.cursor_file, + post.cursor_line, + ) + .await + .map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, e))?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::to_string(&CodeEditResponse { edited_code }).unwrap(), + )) + .unwrap()) +} diff --git a/refact-agent/engine/src/http/routers/v1/knowledge_graph.rs b/refact-agent/engine/src/http/routers/v1/knowledge_graph.rs new file mode 100644 index 000000000..6fecce56f --- /dev/null +++ b/refact-agent/engine/src/http/routers/v1/knowledge_graph.rs @@ -0,0 +1,172 @@ +use axum::Extension; +use axum::response::Result; +use hyper::{Body, Response, StatusCode}; +use serde::Serialize; + +use crate::custom_error::ScratchError; +use crate::global_context::SharedGlobalContext; +use crate::knowledge_graph::build_knowledge_graph; + +#[derive(Serialize)] +struct KgNodeJson { + id: String, + node_type: String, + label: String, +} + +#[derive(Serialize)] +struct KgEdgeJson { + source: String, + target: String, + edge_type: String, +} + +#[derive(Serialize)] +struct KgStatsJson { + doc_count: usize, + tag_count: usize, + file_count: usize, + entity_count: usize, + edge_count: usize, + active_docs: usize, + deprecated_docs: usize, + trajectory_count: usize, +} + +#[derive(Serialize)] +struct KnowledgeGraphJson { + nodes: Vec, + edges: Vec, + stats: KgStatsJson, +} + +pub async fn handle_v1_knowledge_graph( + Extension(gcx): Extension, +) -> Result, ScratchError> { + let kg = build_knowledge_graph(gcx).await; + + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + + for (id, doc) in &kg.docs { + let label = doc.frontmatter.title.clone().unwrap_or_else(|| { + doc.path.file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| id.clone()) + }); + let node_type = match doc.frontmatter.status.as_deref() { + Some("deprecated") => "doc_deprecated", + Some("archived") => "doc_archived", + _ => match doc.frontmatter.kind.as_deref() { + Some("trajectory") => "doc_trajectory", + Some("code") => "doc_code", + Some("decision") => "doc_decision", + _ => "doc", + } + }; + nodes.push(KgNodeJson { + id: id.clone(), + node_type: node_type.to_string(), + label, + }); + } + + for (tag, _) in &kg.tag_index { + nodes.push(KgNodeJson { + id: format!("tag:{}", tag), + node_type: "tag".to_string(), + label: tag.clone(), + }); + } + + for (file, _) in &kg.file_index { + let label = std::path::Path::new(file) + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| file.clone()); + nodes.push(KgNodeJson { + id: format!("file:{}", file), + node_type: "file".to_string(), + label, + }); + } + + for (entity, _) in &kg.entity_index { + nodes.push(KgNodeJson { + id: format!("entity:{}", entity), + node_type: "entity".to_string(), + label: entity.clone(), + }); + } + + for (id, doc) in &kg.docs { + for tag in &doc.frontmatter.tags { + edges.push(KgEdgeJson { + source: id.clone(), + target: format!("tag:{}", tag.to_lowercase()), + edge_type: "tagged_with".to_string(), + }); + } + for file in &doc.frontmatter.filenames { + edges.push(KgEdgeJson { + source: id.clone(), + target: format!("file:{}", file), + edge_type: "references_file".to_string(), + }); + } + for entity in &doc.entities { + edges.push(KgEdgeJson { + source: id.clone(), + target: format!("entity:{}", entity), + edge_type: "mentions".to_string(), + }); + } + for link in &doc.frontmatter.links { + if kg.docs.contains_key(link) { + edges.push(KgEdgeJson { + source: id.clone(), + target: link.clone(), + edge_type: "links_to".to_string(), + }); + } + } + if let Some(superseded_by) = &doc.frontmatter.superseded_by { + if kg.docs.contains_key(superseded_by) { + edges.push(KgEdgeJson { + source: id.clone(), + target: superseded_by.clone(), + edge_type: "superseded_by".to_string(), + }); + } + } + } + + let active_docs = kg.docs.values().filter(|d| d.frontmatter.is_active()).count(); + let deprecated_docs = kg.docs.values().filter(|d| d.frontmatter.is_deprecated()).count(); + let trajectory_count = kg.docs.values() + .filter(|d| d.frontmatter.kind.as_deref() == Some("trajectory")) + .count(); + + let stats = KgStatsJson { + doc_count: kg.docs.len(), + tag_count: kg.tag_index.len(), + file_count: kg.file_index.len(), + entity_count: kg.entity_index.len(), + edge_count: edges.len(), + active_docs, + deprecated_docs, + trajectory_count, + }; + + let response = KnowledgeGraphJson { nodes, edges, stats }; + + let json_string = serde_json::to_string_pretty(&response).map_err(|e| { + ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("JSON serialization error: {}", e)) + })?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .body(Body::from(json_string)) + .unwrap()) +} diff --git a/refact-agent/engine/src/http/routers/v1/workspace.rs b/refact-agent/engine/src/http/routers/v1/workspace.rs index 10a13856a..a8f9b964b 100644 --- a/refact-agent/engine/src/http/routers/v1/workspace.rs +++ b/refact-agent/engine/src/http/routers/v1/workspace.rs @@ -23,8 +23,7 @@ pub async fn handle_v1_set_active_group_id( .map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, format!("JSON problem: {}", e)))?; gcx.write().await.active_group_id = Some(post.group_id); - crate::cloud::threads_sub::trigger_threads_subscription_restart(gcx.clone()).await; - + Ok(Response::builder().status(StatusCode::OK).body(Body::from( serde_json::to_string(&serde_json::json!({ "success": true })).unwrap() )).unwrap()) diff --git a/refact-agent/engine/src/knowledge_graph/kg_builder.rs b/refact-agent/engine/src/knowledge_graph/kg_builder.rs new file mode 100644 index 000000000..2b6b8758b --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_builder.rs @@ -0,0 +1,140 @@ +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use regex::Regex; +use tokio::sync::RwLock as ARwLock; +use tracing::info; +use walkdir::WalkDir; + +use crate::file_filter::KNOWLEDGE_FOLDER_NAME; +use crate::files_correction::get_project_dirs; +use crate::files_in_workspace::get_file_text_from_memory_or_disk; +use crate::global_context::GlobalContext; + +use super::kg_structs::{KnowledgeDoc, KnowledgeFrontmatter, KnowledgeGraph}; + +fn extract_entities(content: &str) -> Vec { + let backtick_re = Regex::new(r"`([a-zA-Z_][a-zA-Z0-9_:]*(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)`").unwrap(); + let mut entities: HashSet = HashSet::new(); + + for caps in backtick_re.captures_iter(content) { + let entity = caps.get(1).unwrap().as_str().to_string(); + if entity.len() >= 3 && entity.len() <= 100 { + entities.insert(entity); + } + } + + entities.into_iter().collect() +} + +pub async fn build_knowledge_graph(gcx: Arc>) -> KnowledgeGraph { + let mut graph = KnowledgeGraph::new(); + + let project_dirs = get_project_dirs(gcx.clone()).await; + let knowledge_dirs: Vec = project_dirs.iter() + .map(|d| d.join(KNOWLEDGE_FOLDER_NAME)) + .filter(|d| d.exists()) + .collect(); + + if knowledge_dirs.is_empty() { + info!("knowledge_graph: no .refact_knowledge directories found"); + return graph; + } + + let workspace_files = collect_workspace_files(gcx.clone()).await; + let mut doc_count = 0; + + for knowledge_dir in knowledge_dirs { + for entry in WalkDir::new(&knowledge_dir) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if !path.is_file() { + continue; + } + + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); + if ext != "md" && ext != "mdx" { + continue; + } + + if path.to_string_lossy().contains("/archive/") { + continue; + } + + let path_buf = path.to_path_buf(); + let text = match get_file_text_from_memory_or_disk(gcx.clone(), &path_buf).await { + Ok(t) => t, + Err(_) => continue, + }; + + let (frontmatter, content_start) = KnowledgeFrontmatter::parse(&text); + let content = text[content_start..].to_string(); + let entities = extract_entities(&content); + + let mut validated_filenames = Vec::new(); + for filename in &frontmatter.filenames { + let exists = workspace_files.contains(filename); + if exists { + validated_filenames.push(filename.clone()); + } + graph.get_or_create_file(filename, exists); + } + + let doc = KnowledgeDoc { + path: path_buf, + frontmatter: KnowledgeFrontmatter { + filenames: validated_filenames, + ..frontmatter + }, + content, + entities, + }; + + graph.add_doc(doc); + doc_count += 1; + } + } + + graph.link_docs(); + + let active_count = graph.docs.values().filter(|d| d.frontmatter.is_active()).count(); + let deprecated_count = graph.docs.values().filter(|d| d.frontmatter.is_deprecated()).count(); + let trajectory_count = graph.docs.values() + .filter(|d| d.frontmatter.kind.as_deref() == Some("trajectory")) + .count(); + let code_count = graph.docs.values() + .filter(|d| d.frontmatter.kind.as_deref() == Some("code")) + .count(); + + info!("knowledge_graph: built successfully"); + info!(" Documents: {} total ({} active, {} deprecated, {} trajectories, {} code)", + doc_count, active_count, deprecated_count, trajectory_count, code_count); + info!(" Tags: {}, Files: {}, Entities: {}", + graph.tag_index.len(), graph.file_index.len(), graph.entity_index.len()); + info!(" Graph edges: {}", graph.graph.edge_count()); + + graph +} + +async fn collect_workspace_files(gcx: Arc>) -> HashSet { + let project_dirs = get_project_dirs(gcx.clone()).await; + let mut files = HashSet::new(); + + for dir in project_dirs { + let indexing = crate::files_blocklist::reload_indexing_everywhere_if_needed(gcx.clone()).await; + if let Ok(paths) = crate::files_in_workspace::ls_files(&*indexing, &dir, true) { + for path in paths { + if let Ok(rel) = path.strip_prefix(&dir) { + files.insert(rel.to_string_lossy().to_string()); + } + files.insert(path.to_string_lossy().to_string()); + } + } + } + + files +} + + diff --git a/refact-agent/engine/src/knowledge_graph/kg_cleanup.rs b/refact-agent/engine/src/knowledge_graph/kg_cleanup.rs new file mode 100644 index 000000000..fe03be16e --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_cleanup.rs @@ -0,0 +1,105 @@ +use std::sync::Arc; +use tokio::sync::RwLock as ARwLock; +use tokio::fs; +use tracing::{info, warn}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; + +use crate::global_context::GlobalContext; +use crate::memories::archive_document; +use super::kg_builder::build_knowledge_graph; + +const CLEANUP_INTERVAL_SECS: u64 = 7 * 24 * 60 * 60; +const TRAJECTORY_MAX_AGE_DAYS: i64 = 90; +const STALE_DOC_AGE_DAYS: i64 = 180; + +#[derive(Debug, Serialize, Deserialize, Default)] +struct CleanupState { + last_run: i64, +} + +async fn load_cleanup_state(gcx: Arc>) -> CleanupState { + let cache_dir = gcx.read().await.cache_dir.clone(); + let state_file = cache_dir.join("knowledge_cleanup_state.json"); + + match fs::read_to_string(&state_file).await { + Ok(content) => serde_json::from_str(&content).unwrap_or_default(), + Err(_) => CleanupState::default(), + } +} + +async fn save_cleanup_state(gcx: Arc>, state: &CleanupState) { + let cache_dir = gcx.read().await.cache_dir.clone(); + let state_file = cache_dir.join("knowledge_cleanup_state.json"); + + if let Ok(content) = serde_json::to_string(state) { + let _ = fs::write(&state_file, content).await; + } +} + +pub async fn knowledge_cleanup_background_task(gcx: Arc>) { + loop { + let state = load_cleanup_state(gcx.clone()).await; + let now = Utc::now().timestamp(); + + if now - state.last_run >= CLEANUP_INTERVAL_SECS as i64 { + info!("knowledge_cleanup: running weekly cleanup"); + + match run_cleanup(gcx.clone()).await { + Ok(report) => { + info!("knowledge_cleanup: completed - archived {} trajectories, {} deprecated docs, {} orphan warnings", + report.archived_trajectories, + report.archived_deprecated, + report.orphan_warnings, + ); + } + Err(e) => { + warn!("knowledge_cleanup: failed - {}", e); + } + } + + let new_state = CleanupState { last_run: now }; + save_cleanup_state(gcx.clone(), &new_state).await; + } + + tokio::time::sleep(tokio::time::Duration::from_secs(24 * 60 * 60)).await; + } +} + +#[derive(Debug, Default)] +struct CleanupReport { + archived_trajectories: usize, + archived_deprecated: usize, + orphan_warnings: usize, +} + +async fn run_cleanup(gcx: Arc>) -> Result { + let kg = build_knowledge_graph(gcx.clone()).await; + let staleness = kg.check_staleness(STALE_DOC_AGE_DAYS, TRAJECTORY_MAX_AGE_DAYS); + let mut report = CleanupReport::default(); + + for path in staleness.stale_trajectories { + match archive_document(gcx.clone(), &path).await { + Ok(_) => report.archived_trajectories += 1, + Err(e) => warn!("Failed to archive trajectory {}: {}", path.display(), e), + } + } + + for path in staleness.deprecated_ready_to_archive { + match archive_document(gcx.clone(), &path).await { + Ok(_) => report.archived_deprecated += 1, + Err(e) => warn!("Failed to archive deprecated doc {}: {}", path.display(), e), + } + } + + report.orphan_warnings = staleness.orphan_file_refs.len(); + for (path, missing_files) in &staleness.orphan_file_refs { + info!("knowledge_cleanup: {} references missing files: {:?}", path.display(), missing_files); + } + + for path in &staleness.past_review { + info!("knowledge_cleanup: {} is past review date", path.display()); + } + + Ok(report) +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_query.rs b/refact-agent/engine/src/knowledge_graph/kg_query.rs new file mode 100644 index 000000000..f795cd3f3 --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_query.rs @@ -0,0 +1,124 @@ +use std::collections::{HashMap, HashSet}; + +use super::kg_structs::{KnowledgeDoc, KnowledgeGraph}; + +struct RelatedDoc { + id: String, + score: f64, +} + +impl KnowledgeGraph { + fn find_related(&self, doc_id: &str, max_results: usize) -> Vec { + let Some(doc) = self.docs.get(doc_id) else { + return vec![]; + }; + + let mut scores: HashMap = HashMap::new(); + + for tag in &doc.frontmatter.tags { + for related_id in self.docs_with_tag(tag) { + if related_id != doc_id { + *scores.entry(related_id).or_insert(0.0) += 1.0; + } + } + } + + for filename in &doc.frontmatter.filenames { + for related_id in self.docs_referencing_file(filename) { + if related_id != doc_id { + *scores.entry(related_id).or_insert(0.0) += 2.0; + } + } + } + + for entity in &doc.entities { + for related_id in self.docs_mentioning_entity(entity) { + if related_id != doc_id { + *scores.entry(related_id).or_insert(0.0) += 1.5; + } + } + } + + let mut results: Vec = scores.into_iter() + .filter(|(id, _)| self.docs.get(id).map(|d| d.frontmatter.is_active()).unwrap_or(false)) + .map(|(id, score)| RelatedDoc { id, score }) + .collect(); + + results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + results.truncate(max_results); + results + } + + pub fn expand_search_results(&self, initial_doc_ids: &[String], max_expansion: usize) -> Vec { + let mut all_ids: HashSet = initial_doc_ids.iter().cloned().collect(); + let mut expanded: Vec = vec![]; + + for doc_id in initial_doc_ids { + let related = self.find_related(doc_id, max_expansion); + for rel in related { + if !all_ids.contains(&rel.id) { + all_ids.insert(rel.id.clone()); + expanded.push(rel.id); + } + if expanded.len() >= max_expansion { + break; + } + } + if expanded.len() >= max_expansion { + break; + } + } + + expanded + } + + fn find_similar_docs(&self, tags: &[String], filenames: &[String], entities: &[String]) -> Vec<(String, f64)> { + let mut scores: HashMap = HashMap::new(); + + for tag in tags { + for id in self.docs_with_tag(tag) { + *scores.entry(id).or_insert(0.0) += 1.0; + } + } + + for filename in filenames { + for id in self.docs_referencing_file(filename) { + *scores.entry(id).or_insert(0.0) += 2.0; + } + } + + for entity in entities { + for id in self.docs_mentioning_entity(entity) { + *scores.entry(id).or_insert(0.0) += 1.5; + } + } + + let mut results: Vec<_> = scores.into_iter() + .filter(|(id, _)| self.docs.get(id).map(|d| d.frontmatter.is_active()).unwrap_or(false)) + .collect(); + + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + results + } + + pub fn get_deprecation_candidates(&self, new_doc_tags: &[String], new_doc_filenames: &[String], new_doc_entities: &[String], exclude_id: Option<&str>) -> Vec<&KnowledgeDoc> { + let similar = self.find_similar_docs(new_doc_tags, new_doc_filenames, new_doc_entities); + + similar.into_iter() + .filter(|(id, score)| { + *score >= 2.0 && exclude_id.map(|e| e != id).unwrap_or(true) + }) + .filter_map(|(id, _)| { + let doc = self.docs.get(&id)?; + if doc.frontmatter.is_deprecated() || doc.frontmatter.is_archived() { + return None; + } + if doc.frontmatter.kind_or_default() == "trajectory" { + return None; + } + Some(doc) + }) + .take(10) + .collect() + } +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_staleness.rs b/refact-agent/engine/src/knowledge_graph/kg_staleness.rs new file mode 100644 index 000000000..97fe6e89d --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_staleness.rs @@ -0,0 +1,99 @@ +use std::collections::HashSet; +use std::path::PathBuf; +use chrono::{NaiveDate, Utc}; + +use super::kg_structs::KnowledgeGraph; + +#[derive(Debug, Default)] +pub struct StalenessReport { + pub orphan_file_refs: Vec<(PathBuf, Vec)>, + pub orphan_docs: Vec, + pub stale_by_age: Vec<(PathBuf, i64)>, + pub past_review: Vec, + pub deprecated_ready_to_archive: Vec, + pub stale_trajectories: Vec, +} + +impl KnowledgeGraph { + pub fn check_staleness(&self, max_age_days: i64, trajectory_max_age_days: i64) -> StalenessReport { + let mut report = StalenessReport::default(); + let today = Utc::now().date_naive(); + + for doc in self.docs.values() { + let kind = doc.frontmatter.kind_or_default(); + + if let Some(created) = &doc.frontmatter.created { + if let Ok(created_date) = NaiveDate::parse_from_str(created, "%Y-%m-%d") { + let age_days = (today - created_date).num_days(); + + if kind == "trajectory" && age_days > trajectory_max_age_days { + report.stale_trajectories.push(doc.path.clone()); + continue; + } + + if age_days > max_age_days && doc.frontmatter.is_active() { + report.stale_by_age.push((doc.path.clone(), age_days)); + } + } + } + + if let Some(review_after) = &doc.frontmatter.review_after { + if let Ok(review_date) = NaiveDate::parse_from_str(review_after, "%Y-%m-%d") { + if today > review_date && doc.frontmatter.is_active() { + report.past_review.push(doc.path.clone()); + } + } + } + + if doc.frontmatter.is_deprecated() { + if let Some(deprecated_at) = &doc.frontmatter.deprecated_at { + if let Ok(deprecated_date) = NaiveDate::parse_from_str(deprecated_at, "%Y-%m-%d") { + let days_deprecated = (today - deprecated_date).num_days(); + if days_deprecated > 60 { + report.deprecated_ready_to_archive.push(doc.path.clone()); + } + } + } + } + + let missing_files: Vec = doc.frontmatter.filenames.iter() + .filter(|f| { + self.file_index.get(*f) + .and_then(|idx| self.graph.node_weight(*idx)) + .map(|node| { + if let super::kg_structs::KgNode::FileRef { exists, .. } = node { + !exists + } else { + false + } + }) + .unwrap_or(true) + }) + .cloned() + .collect(); + + if !missing_files.is_empty() && doc.frontmatter.kind_or_default() == "code" { + report.orphan_file_refs.push((doc.path.clone(), missing_files)); + } + } + + let docs_with_links: HashSet = self.docs.values() + .flat_map(|d| d.frontmatter.links.iter()) + .filter_map(|link| self.docs.get(link)) + .map(|d| d.path.clone()) + .collect(); + + for doc in self.docs.values() { + if doc.frontmatter.tags.is_empty() + && doc.frontmatter.filenames.is_empty() + && doc.entities.is_empty() + && !docs_with_links.contains(&doc.path) + && doc.frontmatter.kind_or_default() != "trajectory" + { + report.orphan_docs.push(doc.path.clone()); + } + } + + report + } +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_structs.rs b/refact-agent/engine/src/knowledge_graph/kg_structs.rs new file mode 100644 index 000000000..dab6e6dc5 --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_structs.rs @@ -0,0 +1,333 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use serde::{Deserialize, Serialize}; +use petgraph::graph::{DiGraph, NodeIndex}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct KnowledgeFrontmatter { + #[serde(default)] + pub id: Option, + #[serde(default)] + pub title: Option, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub created: Option, + #[serde(default)] + pub updated: Option, + #[serde(default)] + pub filenames: Vec, + #[serde(default)] + pub links: Vec, + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub status: Option, + #[serde(default)] + pub superseded_by: Option, + #[serde(default)] + pub deprecated_at: Option, + #[serde(default)] + pub review_after: Option, +} + +impl KnowledgeFrontmatter { + pub fn parse(content: &str) -> (Self, usize) { + if !content.starts_with("---") { + return (Self::default(), 0); + } + + let rest = &content[3..]; + let end_marker = rest.find("\n---"); + let Some(end_idx) = end_marker else { + return (Self::default(), 0); + }; + + let yaml_content = &rest[..end_idx]; + let mut end_offset = 3 + end_idx + 4; + if content.len() > end_offset && content.as_bytes().get(end_offset) == Some(&b'\n') { + end_offset += 1; + } + + match serde_yaml::from_str::(yaml_content) { + Ok(fm) => (fm, end_offset), + Err(_) => (Self::default(), 0), + } + } + + pub fn to_yaml(&self) -> String { + let mut lines = vec!["---".to_string()]; + + if let Some(id) = &self.id { + lines.push(format!("id: \"{}\"", id)); + } + if let Some(title) = &self.title { + lines.push(format!("title: \"{}\"", title.replace('"', "\\\""))); + } + if let Some(kind) = &self.kind { + lines.push(format!("kind: {}", kind)); + } + if let Some(created) = &self.created { + lines.push(format!("created: {}", created)); + } + if let Some(updated) = &self.updated { + lines.push(format!("updated: {}", updated)); + } + if let Some(review_after) = &self.review_after { + lines.push(format!("review_after: {}", review_after)); + } + if let Some(status) = &self.status { + lines.push(format!("status: {}", status)); + } + if !self.tags.is_empty() { + let tags_str = self.tags.iter() + .map(|t| format!("\"{}\"", t)) + .collect::>() + .join(", "); + lines.push(format!("tags: [{}]", tags_str)); + } + if !self.filenames.is_empty() { + let files_str = self.filenames.iter() + .map(|f| format!("\"{}\"", f)) + .collect::>() + .join(", "); + lines.push(format!("filenames: [{}]", files_str)); + } + if !self.links.is_empty() { + let links_str = self.links.iter() + .map(|l| format!("\"{}\"", l)) + .collect::>() + .join(", "); + lines.push(format!("links: [{}]", links_str)); + } + if let Some(superseded_by) = &self.superseded_by { + lines.push(format!("superseded_by: \"{}\"", superseded_by)); + } + if let Some(deprecated_at) = &self.deprecated_at { + lines.push(format!("deprecated_at: {}", deprecated_at)); + } + + lines.push("---".to_string()); + lines.join("\n") + } + + pub fn is_active(&self) -> bool { + self.status.as_deref().unwrap_or("active") == "active" + } + + pub fn is_deprecated(&self) -> bool { + self.status.as_deref() == Some("deprecated") + } + + pub fn is_archived(&self) -> bool { + self.status.as_deref() == Some("archived") + } + + pub fn kind_or_default(&self) -> &str { + self.kind.as_deref().unwrap_or(if self.filenames.is_empty() { "domain" } else { "code" }) + } +} + +#[derive(Debug, Clone)] +pub struct KnowledgeDoc { + pub path: PathBuf, + pub frontmatter: KnowledgeFrontmatter, + pub content: String, + pub entities: Vec, +} + +#[derive(Debug, Clone)] +pub enum KgNode { + Doc { id: String }, + Tag, + FileRef { exists: bool }, + Entity, +} + +#[derive(Debug, Clone)] +pub enum KgEdge { + TaggedWith, + ReferencesFile, + LinksTo, + Mentions, + SupersededBy, +} + +pub struct KnowledgeGraph { + pub graph: DiGraph, + pub doc_index: HashMap, + pub doc_path_index: HashMap, + pub tag_index: HashMap, + pub file_index: HashMap, + pub entity_index: HashMap, + pub docs: HashMap, +} + +impl Default for KnowledgeGraph { + fn default() -> Self { + Self::new() + } +} + +impl KnowledgeGraph { + pub fn new() -> Self { + Self { + graph: DiGraph::new(), + doc_index: HashMap::new(), + doc_path_index: HashMap::new(), + tag_index: HashMap::new(), + file_index: HashMap::new(), + entity_index: HashMap::new(), + docs: HashMap::new(), + } + } + + pub fn get_or_create_tag(&mut self, name: &str) -> NodeIndex { + let normalized = name.to_lowercase().trim().to_string(); + if let Some(&idx) = self.tag_index.get(&normalized) { + return idx; + } + let idx = self.graph.add_node(KgNode::Tag); + self.tag_index.insert(normalized, idx); + idx + } + + pub fn get_or_create_file(&mut self, path: &str, exists: bool) -> NodeIndex { + if let Some(&idx) = self.file_index.get(path) { + return idx; + } + let idx = self.graph.add_node(KgNode::FileRef { exists }); + self.file_index.insert(path.to_string(), idx); + idx + } + + pub fn get_or_create_entity(&mut self, name: &str) -> NodeIndex { + if let Some(&idx) = self.entity_index.get(name) { + return idx; + } + let idx = self.graph.add_node(KgNode::Entity); + self.entity_index.insert(name.to_string(), idx); + idx + } + + pub fn add_doc(&mut self, doc: KnowledgeDoc) -> NodeIndex { + let id = doc.frontmatter.id.clone().unwrap_or_else(|| doc.path.to_string_lossy().to_string()); + let path = doc.path.clone(); + + let doc_idx = self.graph.add_node(KgNode::Doc { id: id.clone() }); + self.doc_index.insert(id.clone(), doc_idx); + self.doc_path_index.insert(path, doc_idx); + + for tag in &doc.frontmatter.tags { + let tag_idx = self.get_or_create_tag(tag); + self.graph.add_edge(doc_idx, tag_idx, KgEdge::TaggedWith); + } + + for filename in &doc.frontmatter.filenames { + let file_idx = self.get_or_create_file(filename, true); + self.graph.add_edge(doc_idx, file_idx, KgEdge::ReferencesFile); + } + + for entity in &doc.entities { + let entity_idx = self.get_or_create_entity(entity); + self.graph.add_edge(doc_idx, entity_idx, KgEdge::Mentions); + } + + self.docs.insert(id, doc); + doc_idx + } + + pub fn link_docs(&mut self) { + let links: Vec<(String, String)> = self.docs.iter() + .flat_map(|(id, doc)| { + doc.frontmatter.links.iter().map(|link| (id.clone(), link.clone())).collect::>() + }) + .collect(); + + for (from_id, to_id) in links { + if let (Some(&from_idx), Some(&to_idx)) = (self.doc_index.get(&from_id), self.doc_index.get(&to_id)) { + self.graph.add_edge(from_idx, to_idx, KgEdge::LinksTo); + } + } + + let supersedes: Vec<(String, String)> = self.docs.iter() + .filter_map(|(id, doc)| { + doc.frontmatter.superseded_by.as_ref().map(|s| (id.clone(), s.clone())) + }) + .collect(); + + for (old_id, new_id) in supersedes { + if let (Some(&old_idx), Some(&new_idx)) = (self.doc_index.get(&old_id), self.doc_index.get(&new_id)) { + self.graph.add_edge(old_idx, new_idx, KgEdge::SupersededBy); + } + } + } + + pub fn get_doc_by_id(&self, id: &str) -> Option<&KnowledgeDoc> { + self.docs.get(id) + } + + pub fn get_doc_by_path(&self, path: &PathBuf) -> Option<&KnowledgeDoc> { + self.doc_path_index.get(path) + .and_then(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(*idx) { + self.docs.get(id) + } else { + None + } + }) + } + + pub fn active_docs(&self) -> impl Iterator { + self.docs.values().filter(|d| d.frontmatter.is_active()) + } + + pub fn docs_with_tag(&self, tag: &str) -> HashSet { + let normalized = tag.to_lowercase(); + let Some(&tag_idx) = self.tag_index.get(&normalized) else { + return HashSet::new(); + }; + + self.graph.neighbors_directed(tag_idx, petgraph::Direction::Incoming) + .filter_map(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(idx) { + Some(id.clone()) + } else { + None + } + }) + .collect() + } + + pub fn docs_referencing_file(&self, file_path: &str) -> HashSet { + let Some(&file_idx) = self.file_index.get(file_path) else { + return HashSet::new(); + }; + + self.graph.neighbors_directed(file_idx, petgraph::Direction::Incoming) + .filter_map(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(idx) { + Some(id.clone()) + } else { + None + } + }) + .collect() + } + + pub fn docs_mentioning_entity(&self, entity: &str) -> HashSet { + let Some(&entity_idx) = self.entity_index.get(entity) else { + return HashSet::new(); + }; + + self.graph.neighbors_directed(entity_idx, petgraph::Direction::Incoming) + .filter_map(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(idx) { + Some(id.clone()) + } else { + None + } + }) + .collect() + } +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_subchat.rs b/refact-agent/engine/src/knowledge_graph/kg_subchat.rs new file mode 100644 index 000000000..553ad427d --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_subchat.rs @@ -0,0 +1,209 @@ +use std::sync::Arc; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex as AMutex; + +use crate::at_commands::at_commands::AtCommandsContext; +use crate::call_validation::ChatMessage; +use crate::subchat::subchat_single; + +use super::kg_structs::KnowledgeDoc; + +#[derive(Debug, Serialize, Deserialize)] +pub struct EnrichmentResult { + #[serde(default)] + pub title: Option, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub filenames: Vec, + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub links: Vec, + #[serde(default)] + pub review_after_days: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeprecationDecision { + #[serde(default)] + pub target_id: String, + #[serde(default)] + pub reason: String, + #[serde(default)] + pub confidence: f64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeprecationResult { + #[serde(default)] + pub deprecate: Vec, + #[serde(default)] + pub keep: Vec, +} + +const ENRICHMENT_PROMPT: &str = r#"Analyze the following knowledge content and extract metadata. + +CONTENT: +{content} + +EXTRACTED ENTITIES (backticked identifiers found): +{entities} + +CANDIDATE FILES (from workspace, pick only relevant ones): +{candidate_files} + +CANDIDATE RELATED DOCS (from knowledge base, pick only relevant ones): +{candidate_docs} + +Return a JSON object with: +- title: a concise title for this knowledge (max 80 chars) +- tags: array of relevant tags (lowercase, max 8 tags) +- filenames: array of file paths this knowledge relates to (only from candidates) +- kind: one of "code", "decision", "domain", "process" (based on content) +- links: array of related doc IDs (only from candidates) +- review_after_days: suggested review period (90 for code/decision, 180 for domain) + +JSON only, no explanation:"#; + +const DEPRECATION_PROMPT: &str = r#"A new knowledge document was created. Determine if any existing documents should be deprecated. + +NEW DOCUMENT: +Title: {new_title} +Tags: {new_tags} +Files: {new_files} +Content snippet: {new_snippet} + +CANDIDATE DOCUMENTS TO POTENTIALLY DEPRECATE: +{candidates} + +For each candidate, decide if it should be deprecated because: +- It covers the same topic and the new doc is more complete/updated +- It references the same files with outdated information +- It's a duplicate or near-duplicate + +Return JSON: +{{ + "deprecate": [ + {{"target_id": "...", "reason": "...", "confidence": 0.0-1.0}} + ], + "keep": ["id1", "id2"] +}} + +Only deprecate with confidence >= 0.75. JSON only:"#; + +pub async fn enrich_knowledge_metadata( + ccx: Arc>, + model_id: &str, + content: &str, + entities: &[String], + candidate_files: &[String], + candidate_docs: &[(String, String)], +) -> Result { + let entities_str = entities.join(", "); + let files_str = candidate_files.iter().take(20).cloned().collect::>().join("\n"); + let docs_str = candidate_docs.iter() + .take(10) + .map(|(id, title)| format!("- {}: {}", id, title)) + .collect::>() + .join("\n"); + + let prompt = ENRICHMENT_PROMPT + .replace("{content}", &content.chars().take(2000).collect::()) + .replace("{entities}", &entities_str) + .replace("{candidate_files}", &files_str) + .replace("{candidate_docs}", &docs_str); + + let messages = vec![ChatMessage::new("user".to_string(), prompt)]; + + let results = subchat_single( + ccx, + model_id, + messages, + Some(vec![]), + Some("none".to_string()), + false, + Some(0.0), + Some(1024), + 1, + None, + false, + None, + None, + None, + ).await?; + + let response = results.get(0) + .and_then(|msgs| msgs.last()) + .map(|m| m.content.content_text_only()) + .unwrap_or_default(); + + let json_start = response.find('{').unwrap_or(0); + let json_end = response.rfind('}').map(|i| i + 1).unwrap_or(response.len()); + let json_str = &response[json_start..json_end]; + + serde_json::from_str(json_str).map_err(|e| format!("Failed to parse enrichment JSON: {}", e)) +} + +pub async fn check_deprecation( + ccx: Arc>, + model_id: &str, + new_doc_title: &str, + new_doc_tags: &[String], + new_doc_files: &[String], + new_doc_snippet: &str, + candidates: &[&KnowledgeDoc], +) -> Result { + if candidates.is_empty() { + return Ok(DeprecationResult { deprecate: vec![], keep: vec![] }); + } + + let candidates_str = candidates.iter() + .map(|doc| { + let id = doc.frontmatter.id.clone().unwrap_or_else(|| doc.path.to_string_lossy().to_string()); + let title = doc.frontmatter.title.clone().unwrap_or_default(); + let tags = doc.frontmatter.tags.join(", "); + let files = doc.frontmatter.filenames.join(", "); + let snippet: String = doc.content.chars().take(300).collect(); + format!("ID: {}\nTitle: {}\nTags: {}\nFiles: {}\nSnippet: {}\n---", id, title, tags, files, snippet) + }) + .collect::>() + .join("\n"); + + let prompt = DEPRECATION_PROMPT + .replace("{new_title}", new_doc_title) + .replace("{new_tags}", &new_doc_tags.join(", ")) + .replace("{new_files}", &new_doc_files.join(", ")) + .replace("{new_snippet}", &new_doc_snippet.chars().take(500).collect::()) + .replace("{candidates}", &candidates_str); + + let messages = vec![ChatMessage::new("user".to_string(), prompt)]; + + let results = subchat_single( + ccx, + model_id, + messages, + Some(vec![]), + Some("none".to_string()), + false, + Some(0.0), + Some(1024), + 1, + None, + false, + None, + None, + None, + ).await?; + + let response = results.get(0) + .and_then(|msgs| msgs.last()) + .map(|m| m.content.content_text_only()) + .unwrap_or_default(); + + let json_start = response.find('{').unwrap_or(0); + let json_end = response.rfind('}').map(|i| i + 1).unwrap_or(response.len()); + let json_str = &response[json_start..json_end]; + + serde_json::from_str(json_str).map_err(|e| format!("Failed to parse deprecation JSON: {}", e)) +} diff --git a/refact-agent/engine/src/knowledge_graph/mod.rs b/refact-agent/engine/src/knowledge_graph/mod.rs new file mode 100644 index 000000000..bd434b60b --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/mod.rs @@ -0,0 +1,10 @@ +pub mod kg_structs; +pub mod kg_builder; +pub mod kg_query; +pub mod kg_staleness; +pub mod kg_subchat; +pub mod kg_cleanup; + +pub use kg_structs::KnowledgeFrontmatter; +pub use kg_builder::build_knowledge_graph; +pub use kg_cleanup::knowledge_cleanup_background_task; diff --git a/refact-agent/engine/src/main.rs b/refact-agent/engine/src/main.rs index 86fb58f8d..70fdab5d4 100644 --- a/refact-agent/engine/src/main.rs +++ b/refact-agent/engine/src/main.rs @@ -62,10 +62,10 @@ mod http; mod integrations; mod privacy; mod git; -mod cloud; mod agentic; mod memories; mod files_correction_cache; +mod knowledge_graph; pub mod constants; #[tokio::main] diff --git a/refact-agent/engine/src/memories.rs b/refact-agent/engine/src/memories.rs index 93cea2113..499933a14 100644 --- a/refact-agent/engine/src/memories.rs +++ b/refact-agent/engine/src/memories.rs @@ -1,301 +1,501 @@ use std::path::PathBuf; use std::sync::Arc; -use itertools::Itertools; -use log::error; +use chrono::{Local, Duration}; +use regex::Regex; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; use tokio::sync::RwLock as ARwLock; -use crate::global_context::GlobalContext; +use tokio::sync::Mutex as AMutex; use tokio::fs; -use tokio_rusqlite::Connection; use tracing::{info, warn}; +use uuid::Uuid; +use walkdir::WalkDir; +use crate::at_commands::at_commands::AtCommandsContext; +use crate::file_filter::KNOWLEDGE_FOLDER_NAME; +use crate::files_correction::get_project_dirs; +use crate::files_in_workspace::get_file_text_from_memory_or_disk; +use crate::global_context::{GlobalContext, try_load_caps_quickly_if_not_present}; +use crate::knowledge_graph::kg_structs::KnowledgeFrontmatter; +use crate::knowledge_graph::kg_subchat::{enrich_knowledge_metadata, check_deprecation}; +use crate::knowledge_graph::build_knowledge_graph; +use crate::vecdb::vdb_structs::VecdbSearch; #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct MemoRecord { - pub iknow_id: String, - pub iknow_tags: Vec, - pub iknow_memory: String, + pub memid: String, + pub tags: Vec, + pub content: String, + pub file_path: Option, + pub line_range: Option<(u64, u64)>, + pub title: Option, + pub created: Option, + pub kind: Option, } +fn generate_slug(content: &str) -> String { + let first_line = content.lines().next().unwrap_or("knowledge"); + first_line + .chars() + .filter(|c| c.is_alphanumeric() || c.is_whitespace()) + .collect::() + .split_whitespace() + .take(5) + .collect::>() + .join("-") + .to_lowercase() + .chars() + .take(50) + .collect() +} -pub async fn memories_migration( - gcx: Arc>, - config_dir: PathBuf -) { - // Disable migration for now - if true { - return; - } - - if let None = gcx.read().await.active_group_id.clone() { - info!("No active group set up, skipping memory migration"); - return; - } - - let legacy_db_path = config_dir.join("memories.sqlite"); - if !legacy_db_path.exists() { - return; +fn generate_filename(content: &str) -> String { + let timestamp = Local::now().format("%Y-%m-%d_%H%M%S").to_string(); + let slug = generate_slug(content); + let short_uuid = &Uuid::new_v4().to_string()[..8]; + if slug.is_empty() { + format!("{}_{}_knowledge.md", timestamp, short_uuid) + } else { + format!("{}_{}_{}.md", timestamp, short_uuid, slug) } - - info!("Found legacy memory database at {:?}, starting migration", legacy_db_path); - - let conn = match Connection::open(&legacy_db_path).await { - Ok(conn) => conn, - Err(e) => { - warn!("Failed to open legacy database: {}", e); - return; - } - }; - - let memories: Vec<(String, String)> = match conn.call(|conn| { - // Query all memories - let mut stmt = conn.prepare("SELECT m_type, m_payload FROM memories")?; - let rows = stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - )) - })?; - - let mut memories = Vec::new(); - for row in rows { - memories.push(row?); - } - - Ok(memories.into_iter().unique_by(|(_, m_payload)| m_payload.clone()).collect()) - }).await { - Ok(memories) => memories, - Err(e) => { - warn!("Failed to query memories: {}", e); - return; - } +} + +pub fn create_frontmatter( + title: Option<&str>, + tags: &[String], + filenames: &[String], + links: &[String], + kind: &str, +) -> KnowledgeFrontmatter { + let now = Local::now(); + let created = now.format("%Y-%m-%d").to_string(); + let review_days = match kind { + "trajectory" => 90, + "preference" => 365, + _ => 90, }; - - if memories.is_empty() { - info!("No memories found in legacy database"); - return; + let review_after = (now + Duration::days(review_days)).format("%Y-%m-%d").to_string(); + + KnowledgeFrontmatter { + id: Some(Uuid::new_v4().to_string()), + title: title.map(|t| t.to_string()), + tags: tags.to_vec(), + created: Some(created.clone()), + updated: Some(created), + filenames: filenames.to_vec(), + links: links.to_vec(), + kind: Some(kind.to_string()), + status: Some("active".to_string()), + superseded_by: None, + deprecated_at: None, + review_after: Some(review_after), } - - info!("Found {} memories in legacy database, migrating to cloud", memories.len()); - - // Migrate each memory to the cloud - let mut success_count = 0; - let mut error_count = 0; - for (m_type, m_payload) in memories { - if m_payload.is_empty() { - warn!("Memory payload is empty, skipping"); - continue; - } - match memories_add(gcx.clone(), &m_type, &m_payload, true).await { - Ok(_) => { - success_count += 1; - if success_count % 10 == 0 { - info!("Migrated {} memories so far", success_count); - } - }, - Err(e) => { - error_count += 1; - warn!("Failed to migrate memory: {}", e); - } - } +} + +async fn get_knowledge_dir(gcx: Arc>) -> Result { + let project_dirs = get_project_dirs(gcx).await; + let workspace_root = project_dirs.first().ok_or("No workspace folder found")?; + Ok(workspace_root.join(KNOWLEDGE_FOLDER_NAME)) +} + +pub async fn memories_add( + gcx: Arc>, + frontmatter: &KnowledgeFrontmatter, + content: &str, +) -> Result { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + fs::create_dir_all(&knowledge_dir).await.map_err(|e| format!("Failed to create knowledge dir: {}", e))?; + + let filename = generate_filename(content); + let file_path = knowledge_dir.join(&filename); + + if file_path.exists() { + return Err(format!("File already exists: {}", file_path.display())); } - - info!("Memory migration complete: {} succeeded, {} failed", success_count, error_count); - if success_count > 0 { - match fs::remove_file(legacy_db_path.clone()).await { - Ok(_) => info!("Removed legacy database: {:?}", legacy_db_path), - Err(e) => warn!("Failed to remove legacy database: {}", e), - } + + let md_content = format!("{}\n\n{}", frontmatter.to_yaml(), content); + fs::write(&file_path, &md_content).await.map_err(|e| format!("Failed to write knowledge file: {}", e))?; + + info!("Created knowledge entry: {}", file_path.display()); + + if let Some(vecdb) = gcx.read().await.vec_db.lock().await.as_ref() { + vecdb.vectorizer_enqueue_files(&vec![file_path.to_string_lossy().to_string()], true).await; } + + Ok(file_path) } -pub async fn memories_add( + + +pub async fn memories_search( gcx: Arc>, - m_type: &str, - m_memory: &str, - _unknown_project: bool -) -> Result<(), String> { - let client = reqwest::Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let active_group_id = gcx.read().await.active_group_id.clone() - .ok_or("active_group_id must be set")?; - let query = r#" - mutation CreateKnowledgeItem($input: FKnowledgeItemInput!) { - knowledge_item_create(input: $input) { - iknow_id + query: &str, + top_n: usize, +) -> Result, String> { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + + let has_vecdb = gcx.read().await.vec_db.lock().await.is_some(); + if has_vecdb { + let vecdb_lock = gcx.read().await.vec_db.clone(); + let vecdb_guard = vecdb_lock.lock().await; + let vecdb = vecdb_guard.as_ref().unwrap(); + let search_result = vecdb.vecdb_search(query.to_string(), top_n * 3, None).await + .map_err(|e| format!("VecDB search failed: {}", e))?; + + let mut records = Vec::new(); + for rec in search_result.results { + let path_str = rec.file_path.to_string_lossy().to_string(); + if !path_str.contains(KNOWLEDGE_FOLDER_NAME) { + continue; } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .header("User-Agent", "refact-lsp") - .json(&json!({ - "query": query, - "variables": { - "input": { - "iknow_memory": m_memory, - "located_fgroup_id": active_group_id, - "iknow_is_core": false, - "iknow_tags": vec![m_type.to_string()], - "owner_shared": false - } + + let text = match get_file_text_from_memory_or_disk(gcx.clone(), &rec.file_path).await { + Ok(t) => t, + Err(_) => continue, + }; + + let (frontmatter, _content_start) = KnowledgeFrontmatter::parse(&text); + + if frontmatter.is_archived() { + continue; } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(_) = data.get("knowledge_item_create") { - info!("Successfully added memory to remote server"); - return Ok(()); + + let lines: Vec<&str> = text.lines().collect(); + let start = (rec.start_line as usize).min(lines.len().saturating_sub(1)); + let end = (rec.end_line as usize).min(lines.len().saturating_sub(1)); + let snippet = lines[start..=end].join("\n"); + + let id = frontmatter.id.clone().unwrap_or_else(|| path_str.clone()); + + records.push(MemoRecord { + memid: format!("{}:{}-{}", id, rec.start_line, rec.end_line), + tags: frontmatter.tags, + content: snippet, + file_path: Some(rec.file_path.clone()), + line_range: Some((rec.start_line, rec.end_line)), + title: frontmatter.title, + created: frontmatter.created, + kind: frontmatter.kind, + }); + + if records.len() >= top_n { + break; } } - Err("Failed to add memory to remote server".to_string()) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!("Failed to add memory to remote server: HTTP status {}, error: {}", status, error_text)) + + if !records.is_empty() { + return Ok(records); + } } + + memories_search_fallback(gcx, query, top_n, &knowledge_dir).await } -pub async fn memories_search( +async fn memories_search_fallback( gcx: Arc>, - q: &String, + query: &str, top_n: usize, + knowledge_dir: &PathBuf, ) -> Result, String> { - let client = reqwest::Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let active_group_id = gcx.read().await.active_group_id.clone() - .ok_or("active_group_id must be set")?; - let query = r#" - query KnowledgeSearch($fgroup_id: String!, $q: String!, $top_n: Int!) { - knowledge_vecdb_search(fgroup_id: $fgroup_id, q: $q, top_n: $top_n) { - iknow_id - iknow_memory - iknow_tags - } + let query_lower = query.to_lowercase(); + let query_words: Vec<&str> = query_lower.split_whitespace().collect(); + let mut scored_results: Vec<(usize, MemoRecord)> = Vec::new(); + + if !knowledge_dir.exists() { + return Ok(vec![]); + } + + for entry in WalkDir::new(knowledge_dir).into_iter().filter_map(|e| e.ok()) { + let path = entry.path(); + if !path.is_file() { + continue; } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .header("User-Agent", "refact-lsp") - .json(&json!({ - "query": query, - "variables": { - "fgroup_id": active_group_id, - "q": q, - "top_n": top_n, - } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); + if path.to_string_lossy().contains("/archive/") { + continue; } - if let Some(data) = response_json.get("data") { - if let Some(memories_value) = data.get("knowledge_vecdb_search") { - let memories: Vec = serde_json::from_value(memories_value.clone()) - .map_err(|e| format!("Failed to parse expert: {}", e))?; - return Ok(memories); - } + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); + if ext != "md" && ext != "mdx" { + continue; } - Err("Failed to get memories from remote server".to_string()) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!("Failed to get memories from remote server: HTTP status {}, error: {}", status, error_text)) + + let text = match get_file_text_from_memory_or_disk(gcx.clone(), &path.to_path_buf()).await { + Ok(t) => t, + Err(_) => continue, + }; + + let text_lower = text.to_lowercase(); + let score: usize = query_words.iter().filter(|w| text_lower.contains(*w)).count(); + if score == 0 { + continue; + } + + let (frontmatter, content_start) = KnowledgeFrontmatter::parse(&text); + if frontmatter.is_archived() { + continue; + } + + let id = frontmatter.id.clone().unwrap_or_else(|| path.to_string_lossy().to_string()); + let content_preview: String = text[content_start..].chars().take(500).collect(); + + scored_results.push((score, MemoRecord { + memid: id, + tags: frontmatter.tags, + content: content_preview, + file_path: Some(path.to_path_buf()), + line_range: None, + title: frontmatter.title, + created: frontmatter.created, + kind: frontmatter.kind, + })); } + + scored_results.sort_by(|a, b| b.0.cmp(&a.0)); + Ok(scored_results.into_iter().take(top_n).map(|(_, r)| r).collect()) } -pub async fn memories_get_core( - gcx: Arc> -) -> Result, String> { - let client = reqwest::Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let active_group_id = gcx.read().await.active_group_id.clone() - .ok_or("active_group_id must be set")?; - let query = r#" - query KnowledgeSearch($fgroup_id: String!) { - knowledge_get_cores(fgroup_id: $fgroup_id) { - iknow_id - iknow_memory - iknow_tags - } +pub async fn save_trajectory( + gcx: Arc>, + compressed_trajectory: &str, +) -> Result { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + let trajectories_dir = knowledge_dir.join("trajectories"); + fs::create_dir_all(&trajectories_dir).await.map_err(|e| format!("Failed to create trajectories dir: {}", e))?; + + let filename = generate_filename(compressed_trajectory); + let file_path = trajectories_dir.join(&filename); + + let frontmatter = create_frontmatter( + compressed_trajectory.lines().next(), + &["trajectory".to_string()], + &[], + &[], + "trajectory", + ); + + let md_content = format!("{}\n\n{}", frontmatter.to_yaml(), compressed_trajectory); + fs::write(&file_path, &md_content).await.map_err(|e| format!("Failed to write trajectory file: {}", e))?; + + info!("Saved trajectory: {}", file_path.display()); + + if let Some(vecdb) = gcx.read().await.vec_db.lock().await.as_ref() { + vecdb.vectorizer_enqueue_files(&vec![file_path.to_string_lossy().to_string()], true).await; + } + + let _ = build_knowledge_graph(gcx).await; + + Ok(file_path) +} + +pub async fn deprecate_document( + gcx: Arc>, + doc_path: &PathBuf, + superseded_by: Option<&str>, + reason: &str, +) -> Result<(), String> { + let text = get_file_text_from_memory_or_disk(gcx.clone(), doc_path).await + .map_err(|e| format!("Failed to read document: {}", e))?; + + let (mut frontmatter, content_start) = KnowledgeFrontmatter::parse(&text); + let content = &text[content_start..]; + + frontmatter.status = Some("deprecated".to_string()); + frontmatter.deprecated_at = Some(Local::now().format("%Y-%m-%d").to_string()); + if let Some(new_id) = superseded_by { + frontmatter.superseded_by = Some(new_id.to_string()); + } + + let deprecated_banner = format!("\n\n> ⚠️ **DEPRECATED**: {}\n", reason); + let new_content = format!("{}\n{}{}", frontmatter.to_yaml(), deprecated_banner, content); + + fs::write(doc_path, new_content).await.map_err(|e| format!("Failed to write: {}", e))?; + + info!("Deprecated document: {}", doc_path.display()); + + if let Some(vecdb) = gcx.read().await.vec_db.lock().await.as_ref() { + vecdb.vectorizer_enqueue_files(&vec![doc_path.to_string_lossy().to_string()], true).await; + } + + Ok(()) +} + +pub async fn archive_document(gcx: Arc>, doc_path: &PathBuf) -> Result { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + let archive_dir = knowledge_dir.join("archive"); + fs::create_dir_all(&archive_dir).await.map_err(|e| format!("Failed to create archive dir: {}", e))?; + + let filename = doc_path.file_name().ok_or("Invalid filename")?; + let archive_path = archive_dir.join(filename); + + fs::rename(doc_path, &archive_path).await.map_err(|e| format!("Failed to move to archive: {}", e))?; + + info!("Archived document: {} -> {}", doc_path.display(), archive_path.display()); + + Ok(archive_path) +} + +fn extract_entities(content: &str) -> Vec { + let backtick_re = Regex::new(r"`([a-zA-Z_][a-zA-Z0-9_:]*(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)`").unwrap(); + backtick_re.captures_iter(content) + .map(|c| c.get(1).unwrap().as_str().to_string()) + .filter(|e| e.len() >= 3 && e.len() <= 100) + .collect() +} + +fn extract_file_paths(content: &str) -> Vec { + let path_re = Regex::new(r"(?:^|[\s`])((?:[a-zA-Z0-9_-]+/)+[a-zA-Z0-9_-]+\.[a-zA-Z0-9]+)").unwrap(); + path_re.captures_iter(content) + .map(|c| c.get(1).unwrap().as_str().to_string()) + .collect() +} + +pub struct EnrichmentParams { + pub base_tags: Vec, + pub base_filenames: Vec, + pub base_kind: String, + pub base_title: Option, +} + +pub async fn memories_add_enriched( + ccx: Arc>, + content: &str, + params: EnrichmentParams, +) -> Result { + let gcx = ccx.lock().await.global_context.clone(); + + let entities = extract_entities(content); + let detected_paths = extract_file_paths(content); + + let caps = try_load_caps_quickly_if_not_present(gcx.clone(), 0).await + .map_err(|e| format!("Failed to load caps: {}", e.message))?; + let light_model = if caps.defaults.chat_light_model.is_empty() { + caps.defaults.chat_default_model.clone() + } else { + caps.defaults.chat_light_model.clone() + }; + + let kg = build_knowledge_graph(gcx.clone()).await; + + let candidate_files: Vec = { + let mut files = params.base_filenames.clone(); + files.extend(detected_paths); + files.into_iter().take(30).collect() + }; + + let candidate_docs: Vec<(String, String)> = kg.active_docs() + .take(20) + .map(|d| { + let id = d.frontmatter.id.clone().unwrap_or_else(|| d.path.to_string_lossy().to_string()); + let title = d.frontmatter.title.clone().unwrap_or_else(|| "Untitled".to_string()); + (id, title) + }) + .collect(); + + let enrichment = enrich_knowledge_metadata( + ccx.clone(), + &light_model, + content, + &entities, + &candidate_files, + &candidate_docs, + ).await; + + let (final_title, final_tags, final_filenames, final_kind, final_links, review_days) = match enrichment { + Ok(e) => { + let mut tags = params.base_tags.clone(); + tags.extend(e.tags); + tags.sort(); + tags.dedup(); + + let mut files = params.base_filenames.clone(); + files.extend(e.filenames); + files.sort(); + files.dedup(); + + let kind = e.kind.unwrap_or_else(|| params.base_kind.clone()); + + ( + e.title.or(params.base_title.clone()).or_else(|| content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())), + if tags.is_empty() { vec![params.base_kind.clone()] } else { tags }, + files, + kind, + e.links, + e.review_after_days.unwrap_or(90), + ) } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .header("User-Agent", "refact-lsp") - .json(&json!({ - "query": query, - "variables": { - "fgroup_id": active_group_id - } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); + Err(e) => { + warn!("Enrichment failed, using defaults: {}", e); + let tags = if params.base_tags.is_empty() { vec![params.base_kind.clone()] } else { params.base_tags }; + ( + params.base_title.or_else(|| content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())), + tags, + params.base_filenames, + params.base_kind, + vec![], + 90, + ) } - if let Some(data) = response_json.get("data") { - if let Some(memories_value) = data.get("knowledge_get_cores") { - let memories: Vec = serde_json::from_value(memories_value.clone()) - .map_err(|e| format!("Failed to parse expert: {}", e))?; - return Ok(memories); + }; + + let now = Local::now(); + let frontmatter = KnowledgeFrontmatter { + id: Some(Uuid::new_v4().to_string()), + title: final_title.clone(), + tags: final_tags.clone(), + created: Some(now.format("%Y-%m-%d").to_string()), + updated: Some(now.format("%Y-%m-%d").to_string()), + filenames: final_filenames.clone(), + links: final_links, + kind: Some(final_kind), + status: Some("active".to_string()), + superseded_by: None, + deprecated_at: None, + review_after: Some((now + Duration::days(review_days)).format("%Y-%m-%d").to_string()), + }; + + let file_path = memories_add(gcx.clone(), &frontmatter, content).await?; + let new_doc_id = frontmatter.id.clone().unwrap(); + + let deprecation_candidates = kg.get_deprecation_candidates( + &final_tags, + &final_filenames, + &entities, + Some(&new_doc_id), + ); + + if !deprecation_candidates.is_empty() { + let snippet: String = content.chars().take(500).collect(); + + match check_deprecation( + ccx.clone(), + &light_model, + final_title.as_deref().unwrap_or("Untitled"), + &final_tags, + &final_filenames, + &snippet, + &deprecation_candidates, + ).await { + Ok(result) => { + for decision in result.deprecate { + if decision.confidence >= 0.75 { + if let Some(doc) = kg.get_doc_by_id(&decision.target_id) { + if let Err(e) = deprecate_document( + gcx.clone(), + &doc.path, + Some(&new_doc_id), + &decision.reason, + ).await { + warn!("Failed to deprecate {}: {}", decision.target_id, e); + } else { + info!("Deprecated {} (confidence: {:.2}): {}", decision.target_id, decision.confidence, decision.reason); + } + } + } + } + } + Err(e) => { + warn!("Deprecation check failed: {}", e); } } - Err("Failed to get core memories from remote server".to_string()) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!("Failed to get core memories from remote server: HTTP status {}, error: {}", status, error_text)) } + + Ok(file_path) } diff --git a/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs b/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs index 45b80d46f..63d214625 100644 --- a/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs +++ b/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs @@ -216,22 +216,10 @@ pub async fn system_prompt_add_extra_instructions( } if system_prompt.contains("%KNOWLEDGE_INSTRUCTIONS%") { if include_project_info { - let active_group_id = gcx.read().await.active_group_id.clone(); - if active_group_id.is_some() { - let cfg = crate::yaml_configs::customization_loader::load_customization_compiled_in(); - let mut knowledge_instructions = cfg.get("KNOWLEDGE_INSTRUCTIONS_META") - .map(|x| x.as_str().unwrap_or("").to_string()).unwrap_or("".to_string()); - if let Some(core_memories) = crate::memories::memories_get_core(gcx.clone()).await.ok() { - knowledge_instructions.push_str("\nThere are some pre-existing core memories:\n"); - for mem in core_memories { - knowledge_instructions.push_str(&format!("🗃️\n{}\n\n", mem.iknow_memory)); - } - } - system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", &knowledge_instructions); - tracing::info!("adding up extra knowledge instructions"); - } else { - system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", ""); - } + let cfg = crate::yaml_configs::customization_loader::load_customization_compiled_in(); + let knowledge_instructions = cfg.get("KNOWLEDGE_INSTRUCTIONS_META") + .map(|x| x.as_str().unwrap_or("").to_string()).unwrap_or("".to_string()); + system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", &knowledge_instructions); } else { system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", ""); } diff --git a/refact-agent/engine/src/tools/file_edit/auxiliary.rs b/refact-agent/engine/src/tools/file_edit/auxiliary.rs index a7036182f..d72b400be 100644 --- a/refact-agent/engine/src/tools/file_edit/auxiliary.rs +++ b/refact-agent/engine/src/tools/file_edit/auxiliary.rs @@ -169,6 +169,8 @@ pub async fn write_file(gcx: Arc>, path: &PathBuf, file_t warn!("{err}"); err })?; + // Invalidate stale cache entry so subsequent reads get fresh content from disk + gcx.write().await.documents_state.memory_document_map.remove(path); } Ok((before_text, file_text.to_string())) diff --git a/refact-agent/engine/src/tools/tool_create_knowledge.rs b/refact-agent/engine/src/tools/tool_create_knowledge.rs index 0d14e978f..ad4085f0b 100644 --- a/refact-agent/engine/src/tools/tool_create_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_create_knowledge.rs @@ -8,6 +8,7 @@ use tokio::sync::Mutex as AMutex; use crate::at_commands::at_commands::AtCommandsContext; use crate::call_validation::{ChatMessage, ChatContent, ContextEnum}; use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType}; +use crate::memories::{memories_add_enriched, EnrichmentParams}; pub struct ToolCreateKnowledge { pub config_path: String, @@ -27,17 +28,25 @@ impl Tool for ToolCreateKnowledge { }, agentic: true, experimental: false, - description: "Creates a new knowledge entry in the vector database to help with future tasks.".to_string(), + description: "Creates a new knowledge entry. Uses AI to enrich metadata and check for outdated documents.".to_string(), parameters: vec![ ToolParam { - name: "knowledge_entry".to_string(), + name: "content".to_string(), param_type: "string".to_string(), - description: "The detailed knowledge content to store. Include comprehensive information about implementation details, code patterns, architectural decisions, troubleshooting steps, or solution approaches. Document what you did, how you did it, why you made certain choices, and any important observations or lessons learned. This field should contain the rich, detailed content that future searches will retrieve.".to_string(), - } - ], - parameters_required: vec![ - "knowledge_entry".to_string(), + description: "The knowledge content to store.".to_string(), + }, + ToolParam { + name: "tags".to_string(), + param_type: "string".to_string(), + description: "Comma-separated tags (optional, will be auto-enriched).".to_string(), + }, + ToolParam { + name: "filenames".to_string(), + param_type: "string".to_string(), + description: "Comma-separated related file paths (optional, will be auto-enriched).".to_string(), + }, ], + parameters_required: vec!["content".to_string()], } } @@ -47,33 +56,42 @@ impl Tool for ToolCreateKnowledge { tool_call_id: &String, args: &HashMap, ) -> Result<(bool, Vec), String> { - info!("run @create-knowledge with args: {:?}", args); - let gcx = { - let ccx_locked = ccx.lock().await; - ccx_locked.global_context.clone() - }; - let knowledge_entry = match args.get("knowledge_entry") { + info!("create_knowledge {:?}", args); + + let content = match args.get("content") { Some(Value::String(s)) => s.clone(), - Some(v) => return Err(format!("argument `knowledge_entry` is not a string: {:?}", v)), - None => return Err("argument `knowledge_entry` is missing".to_string()) + Some(v) => return Err(format!("argument `content` is not a string: {:?}", v)), + None => return Err("argument `content` is missing".to_string()), + }; + + let user_tags: Vec = match args.get("tags") { + Some(Value::String(s)) => s.split(',').map(|t| t.trim().to_string()).filter(|t| !t.is_empty()).collect(), + _ => vec![], + }; + + let user_filenames: Vec = match args.get("filenames") { + Some(Value::String(s)) => s.split(',').map(|f| f.trim().to_string()).filter(|f| !f.is_empty()).collect(), + _ => vec![], + }; + + let enrichment_params = EnrichmentParams { + base_tags: user_tags, + base_filenames: user_filenames, + base_kind: "knowledge".to_string(), + base_title: None, }; - crate::memories::memories_add( - gcx.clone(), - "knowledge-entry", - &knowledge_entry, - false - ).await.map_err(|e| format!("Failed to store knowledge: {e}"))?; - let mut results = vec![]; - results.push(ContextEnum::ChatMessage(ChatMessage { + let file_path = memories_add_enriched(ccx.clone(), &content, enrichment_params).await?; + + let result_msg = format!("Knowledge entry created: {}", file_path.display()); + + Ok((false, vec![ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), - content: ChatContent::SimpleText("Knowledge entry created successfully".to_string()), + content: ChatContent::SimpleText(result_msg), tool_calls: None, tool_call_id: tool_call_id.clone(), ..Default::default() - })); - - Ok((false, results)) + })])) } fn tool_depends_on(&self) -> Vec { diff --git a/refact-agent/engine/src/tools/tool_create_memory_bank.rs b/refact-agent/engine/src/tools/tool_create_memory_bank.rs index ef88365b0..c926b3c85 100644 --- a/refact-agent/engine/src/tools/tool_create_memory_bank.rs +++ b/refact-agent/engine/src/tools/tool_create_memory_bank.rs @@ -336,7 +336,7 @@ const MB_SYSTEM_PROMPT: &str = r###"• Objective: • Operational Constraint: – Do NOT call create_knowledge() until instructed."###; -const MB_EXPERT_WRAP_UP: &str = r###"Call create_knowledge() now with your complete and full analysis from the previous step if you haven't called it yet. Otherwise just type "Finished"."###; +const MB_EXPERT_WRAP_UP: &str = r###"Call create_knowledge(tags, content) now with your complete and full analysis from the previous step if you haven't called it yet. Use appropriate tags like ["architecture", "module-name", "patterns"]. Otherwise just type "Finished"."###; impl ToolCreateMemoryBank { fn build_step_prompt( @@ -497,8 +497,8 @@ impl Tool for ToolCreateMemoryBank { config_path: self.config_path.clone(), }, agentic: true, - experimental: true, - description: "Gathers information about the project structure (modules, file relations, classes, etc.) and saves this data into the memory bank.".into(), + experimental: false, + description: "Gathers information about the project structure (modules, file relations, classes, etc.) and saves this data into the knowledge base.".into(), parameters: Vec::new(), parameters_required: Vec::new(), } diff --git a/refact-agent/engine/src/tools/tool_deep_research.rs b/refact-agent/engine/src/tools/tool_deep_research.rs index d75b2c38d..0a12842d0 100644 --- a/refact-agent/engine/src/tools/tool_deep_research.rs +++ b/refact-agent/engine/src/tools/tool_deep_research.rs @@ -9,6 +9,7 @@ use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, Too use crate::call_validation::{ChatMessage, ChatContent, ChatUsage, ContextEnum, SubchatParameters}; use crate::at_commands::at_commands::AtCommandsContext; use crate::integrations::integr_abstract::IntegrationConfirmation; +use crate::memories::{memories_add_enriched, EnrichmentParams}; pub struct ToolDeepResearch { pub config_path: String, @@ -191,6 +192,23 @@ impl Tool for ToolDeepResearch { let final_message = format!("# Deep Research Report\n\n{}", research_result.content.content_text_only()); tracing::info!("Deep research completed"); + let title = if research_query.len() > 80 { + format!("{}...", &research_query[..80]) + } else { + research_query.clone() + }; + let enrichment_params = EnrichmentParams { + base_tags: vec!["research".to_string(), "deep-research".to_string()], + base_filenames: vec![], + base_kind: "research".to_string(), + base_title: Some(title), + }; + if let Err(e) = memories_add_enriched(ccx.clone(), &final_message, enrichment_params).await { + tracing::warn!("Failed to create enriched memory from deep research: {}", e); + } else { + tracing::info!("Created enriched memory from deep research"); + } + let mut results = vec![]; results.push(ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), diff --git a/refact-agent/engine/src/tools/tool_knowledge.rs b/refact-agent/engine/src/tools/tool_knowledge.rs index 327a5622d..badeb0f06 100644 --- a/refact-agent/engine/src/tools/tool_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_knowledge.rs @@ -1,21 +1,21 @@ use std::sync::Arc; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use serde_json::Value; use tracing::info; use tokio::sync::Mutex as AMutex; use async_trait::async_trait; +use std::collections::HashMap; use crate::at_commands::at_commands::AtCommandsContext; use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType}; use crate::call_validation::{ChatMessage, ChatContent, ContextEnum}; use crate::memories::memories_search; - +use crate::knowledge_graph::build_knowledge_graph; pub struct ToolGetKnowledge { pub config_path: String, } - #[async_trait] impl Tool for ToolGetKnowledge { fn as_any(&self) -> &dyn std::any::Any { self } @@ -30,12 +30,12 @@ impl Tool for ToolGetKnowledge { }, agentic: true, experimental: false, - description: "Fetches successful trajectories to help you accomplish your task. Call each time you have a new task to increase your chances of success.".to_string(), + description: "Searches project knowledge base for relevant information. Uses semantic search and knowledge graph expansion.".to_string(), parameters: vec![ ToolParam { name: "search_key".to_string(), param_type: "string".to_string(), - description: "Search keys for the knowledge database. Write combined elements from all fields (tools, project components, objectives, and language/framework). This field is used for vector similarity search.".to_string(), + description: "Search query for the knowledge database.".to_string(), } ], parameters_required: vec!["search_key".to_string()], @@ -48,46 +48,94 @@ impl Tool for ToolGetKnowledge { tool_call_id: &String, args: &HashMap, ) -> Result<(bool, Vec), String> { - info!("run @get-knowledge {:?}", args); + info!("knowledge search {:?}", args); - let (gcx, _top_n) = { - let ccx_locked = ccx.lock().await; - (ccx_locked.global_context.clone(), ccx_locked.top_n) - }; + let gcx = ccx.lock().await.global_context.clone(); let search_key = match args.get("search_key") { Some(Value::String(s)) => s.clone(), - Some(v) => { return Err(format!("argument `search_key` is not a string: {:?}", v)) }, - None => { return Err("argument `search_key` is missing".to_string()) } + Some(v) => return Err(format!("argument `search_key` is not a string: {:?}", v)), + None => return Err("argument `search_key` is missing".to_string()), }; - let mem_top_n = 5; - let memories = memories_search(gcx.clone(), &search_key, mem_top_n).await?; - + let memories = memories_search(gcx.clone(), &search_key, 5).await?; + let mut seen_memids = HashSet::new(); - let unique_memories: Vec<_> = memories.into_iter() - .filter(|m| seen_memids.insert(m.iknow_id.clone())) + let mut unique_memories: Vec<_> = memories.into_iter() + .filter(|m| seen_memids.insert(m.memid.clone())) .collect(); - let memories_str = unique_memories.iter().map(|m| { - let payload: String = m.iknow_memory.clone(); - let mut combined = String::new(); - combined.push_str(&format!("🗃️{}\n", m.iknow_id)); - combined.push_str(&payload); - combined.push_str("\n\n"); - combined - }).collect::(); - - let mut results = vec![]; - results.push(ContextEnum::ChatMessage(ChatMessage { + if !unique_memories.is_empty() { + let kg = build_knowledge_graph(gcx.clone()).await; + + let initial_ids: Vec = unique_memories.iter() + .filter_map(|m| m.file_path.as_ref()) + .filter_map(|p| kg.get_doc_by_path(p)) + .filter_map(|d| d.frontmatter.id.clone()) + .collect(); + + let expanded_ids = kg.expand_search_results(&initial_ids, 3); + + for id in expanded_ids { + if let Some(doc) = kg.get_doc_by_id(&id) { + if doc.frontmatter.is_active() && !seen_memids.contains(&id) { + seen_memids.insert(id.clone()); + let snippet: String = doc.content.chars().take(500).collect(); + unique_memories.push(crate::memories::MemoRecord { + memid: id, + tags: doc.frontmatter.tags.clone(), + content: snippet, + file_path: Some(doc.path.clone()), + line_range: None, + title: doc.frontmatter.title.clone(), + created: doc.frontmatter.created.clone(), + kind: doc.frontmatter.kind.clone(), + }); + } + } + } + } + + unique_memories.sort_by(|a, b| { + let a_is_traj = a.kind.as_deref() == Some("trajectory"); + let b_is_traj = b.kind.as_deref() == Some("trajectory"); + a_is_traj.cmp(&b_is_traj) + }); + + let memories_str = if unique_memories.is_empty() { + "No relevant knowledge found.".to_string() + } else { + unique_memories.iter().take(8).map(|m| { + let mut result = String::new(); + if let Some(path) = &m.file_path { + result.push_str(&format!("📄 {}", path.display())); + if let Some((start, end)) = m.line_range { + result.push_str(&format!(":{}-{}", start, end)); + } + result.push('\n'); + } + if let Some(title) = &m.title { + result.push_str(&format!("📌 {}\n", title)); + } + if let Some(kind) = &m.kind { + result.push_str(&format!("📦 {}\n", kind)); + } + if !m.tags.is_empty() { + result.push_str(&format!("🏷️ {}\n", m.tags.join(", "))); + } + result.push_str(&m.content); + result.push_str("\n\n---\n"); + result + }).collect() + }; + + Ok((false, vec![ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), content: ChatContent::SimpleText(memories_str), tool_calls: None, tool_call_id: tool_call_id.clone(), ..Default::default() - })); - - Ok((false, results)) + })])) } fn tool_depends_on(&self) -> Vec { diff --git a/refact-agent/engine/src/tools/tool_mv.rs b/refact-agent/engine/src/tools/tool_mv.rs index 2e4d4dd90..2c195b50c 100644 --- a/refact-agent/engine/src/tools/tool_mv.rs +++ b/refact-agent/engine/src/tools/tool_mv.rs @@ -156,12 +156,26 @@ impl Tool for ToolMv { if dst_metadata.is_dir() { fs::remove_dir_all(&dst_true_path).await .map_err(|e| format!("Failed to remove existing directory '{}': {}", dst_str, e))?; + // Invalidate cache entries for all files under the removed directory + { + let mut gcx_write = gcx.write().await; + let paths_to_remove: Vec<_> = gcx_write.documents_state.memory_document_map + .keys() + .filter(|p| p.starts_with(&dst_true_path)) + .cloned() + .collect(); + for p in paths_to_remove { + gcx_write.documents_state.memory_document_map.remove(&p); + } + } } else { if !dst_metadata.is_dir() { dst_file_content = fs::read_to_string(&dst_true_path).await.unwrap_or_else(|_| "".to_string()); } fs::remove_file(&dst_true_path).await .map_err(|e| format!("Failed to remove existing file '{}': {}", dst_str, e))?; + // Invalidate cache entry for the removed file + gcx.write().await.documents_state.memory_document_map.remove(&dst_true_path); } } @@ -179,6 +193,12 @@ impl Tool for ToolMv { match fs::rename(&src_true_path, &dst_true_path).await { Ok(_) => { + // Invalidate cache entries for both source and destination + { + let mut gcx_write = gcx.write().await; + gcx_write.documents_state.memory_document_map.remove(&src_true_path); + gcx_write.documents_state.memory_document_map.remove(&dst_true_path); + } let corrections = src_str != src_corrected_path || dst_str != dst_corrected_path; let mut messages = vec![]; if !src_is_dir && !src_file_content.is_empty() { @@ -235,6 +255,12 @@ impl Tool for ToolMv { .map_err(|e| format!("Failed to copy '{}' to '{}': {}", src_str, dst_str, e))?; fs::remove_file(&src_true_path).await .map_err(|e| format!("Failed to remove source file '{}' after copy: {}", src_str, e))?; + // Invalidate cache entries for both source and destination + { + let mut gcx_write = gcx.write().await; + gcx_write.documents_state.memory_document_map.remove(&src_true_path); + gcx_write.documents_state.memory_document_map.remove(&dst_true_path); + } let mut messages = vec![]; diff --git a/refact-agent/engine/src/tools/tool_rm.rs b/refact-agent/engine/src/tools/tool_rm.rs index 7e15eb66a..522b97c2d 100644 --- a/refact-agent/engine/src/tools/tool_rm.rs +++ b/refact-agent/engine/src/tools/tool_rm.rs @@ -203,6 +203,18 @@ impl Tool for ToolRm { fs::remove_dir_all(&true_path).await.map_err(|e| { format!("Failed to remove directory '{}': {}", corrected_path, e) })?; + // Invalidate cache entries for all files under the deleted directory + { + let mut gcx_write = gcx.write().await; + let paths_to_remove: Vec<_> = gcx_write.documents_state.memory_document_map + .keys() + .filter(|p| p.starts_with(&true_path)) + .cloned() + .collect(); + for p in paths_to_remove { + gcx_write.documents_state.memory_document_map.remove(&p); + } + } messages.push(ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), content: ChatContent::SimpleText(format!("Removed directory '{}'", corrected_path)), @@ -224,6 +236,8 @@ impl Tool for ToolRm { fs::remove_file(&true_path).await.map_err(|e| { format!("Failed to remove file '{}': {}", corrected_path, e) })?; + // Invalidate cache entry for the deleted file + gcx.write().await.documents_state.memory_document_map.remove(&true_path); if !file_content.is_empty() { let diff_chunk = DiffChunk { file_name: corrected_path.clone(), diff --git a/refact-agent/engine/src/tools/tool_strategic_planning.rs b/refact-agent/engine/src/tools/tool_strategic_planning.rs index ec887ae75..c1aa39b19 100644 --- a/refact-agent/engine/src/tools/tool_strategic_planning.rs +++ b/refact-agent/engine/src/tools/tool_strategic_planning.rs @@ -19,6 +19,7 @@ use crate::files_in_workspace::get_file_text_from_memory_or_disk; use crate::global_context::try_load_caps_quickly_if_not_present; use crate::postprocessing::pp_context_files::postprocess_context_files; use crate::tokens::count_text_tokens_with_fallback; +use crate::memories::{memories_add_enriched, EnrichmentParams}; pub struct ToolStrategicPlanning { pub config_path: String, @@ -318,6 +319,22 @@ impl Tool for ToolStrategicPlanning { let (_, initial_solution) = result?; let final_message = format!("# Solution\n{}", initial_solution.content.content_text_only()); tracing::info!("strategic planning response (combined):\n{}", final_message); + + let filenames: Vec = important_paths.iter() + .map(|p| p.to_string_lossy().to_string()) + .collect(); + let enrichment_params = EnrichmentParams { + base_tags: vec!["planning".to_string(), "strategic".to_string()], + base_filenames: filenames, + base_kind: "decision".to_string(), + base_title: Some("Strategic Plan".to_string()), + }; + if let Err(e) = memories_add_enriched(ccx.clone(), &final_message, enrichment_params).await { + tracing::warn!("Failed to create enriched memory from strategic planning: {}", e); + } else { + tracing::info!("Created enriched memory from strategic planning"); + } + let mut results = vec![]; results.push(ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), diff --git a/refact-agent/engine/src/tools/tools_list.rs b/refact-agent/engine/src/tools/tools_list.rs index a7d1f2794..40ffa18aa 100644 --- a/refact-agent/engine/src/tools/tools_list.rs +++ b/refact-agent/engine/src/tools/tools_list.rs @@ -39,19 +39,17 @@ fn tool_available( async fn tool_available_from_gcx( gcx: Arc>, ) -> impl Fn(&Box) -> bool { - let (ast_on, vecdb_on, allow_experimental, active_group_id) = { + let (ast_on, vecdb_on, allow_experimental) = { let gcx_locked = gcx.read().await; let vecdb_on = gcx_locked.vec_db.lock().await.is_some(); - (gcx_locked.ast_service.is_some(), vecdb_on, - gcx_locked.cmdline.experimental, gcx_locked.active_group_id.clone()) + (gcx_locked.ast_service.is_some(), vecdb_on, gcx_locked.cmdline.experimental) }; - let (is_there_a_thinking_model, allow_knowledge) = match try_load_caps_quickly_if_not_present(gcx.clone(), 0).await { - Ok(caps) => { - (caps.chat_models.get(&caps.defaults.chat_thinking_model).is_some(), active_group_id.is_some()) - }, - Err(_) => (false, false), + let is_there_a_thinking_model = match try_load_caps_quickly_if_not_present(gcx.clone(), 0).await { + Ok(caps) => caps.chat_models.get(&caps.defaults.chat_thinking_model).is_some(), + Err(_) => false, }; + let allow_knowledge = true; move |tool: &Box| { tool_available( diff --git a/refact-agent/engine/src/vecdb/mod.rs b/refact-agent/engine/src/vecdb/mod.rs index 57ce015be..e297b1fac 100644 --- a/refact-agent/engine/src/vecdb/mod.rs +++ b/refact-agent/engine/src/vecdb/mod.rs @@ -1,5 +1,6 @@ pub mod vdb_highlev; pub mod vdb_file_splitter; +pub mod vdb_markdown_splitter; pub mod vdb_structs; pub mod vdb_remote; pub mod vdb_sqlite; diff --git a/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs b/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs new file mode 100644 index 000000000..25142acc7 --- /dev/null +++ b/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs @@ -0,0 +1,234 @@ +use std::path::PathBuf; +use std::sync::Arc; +use regex::Regex; +use tokio::sync::RwLock as ARwLock; + +use crate::files_in_workspace::Document; +use crate::global_context::GlobalContext; +use crate::vecdb::vdb_structs::SplitResult; +use crate::ast::chunk_utils::official_text_hashing_function; +use crate::knowledge_graph::KnowledgeFrontmatter; + +pub use crate::knowledge_graph::KnowledgeFrontmatter as MarkdownFrontmatter; + +#[derive(Debug, Clone)] +struct MarkdownSection { + heading_path: Vec, + content: String, + start_line: usize, + end_line: usize, +} + +pub struct MarkdownFileSplitter { + max_tokens: usize, + overlap_lines: usize, +} + +impl MarkdownFileSplitter { + pub fn new(max_tokens: usize) -> Self { + Self { max_tokens, overlap_lines: 3 } + } + + pub async fn split( + &self, + doc: &Document, + gcx: Arc>, + ) -> Result, String> { + let text = doc.clone().get_text_or_read_from_disk(gcx).await.map_err(|e| e.to_string())?; + let path = doc.doc_path.clone(); + let (frontmatter, content_start) = MarkdownFrontmatter::parse(&text); + let frontmatter_lines = if content_start > 0 { text[..content_start].lines().count() } else { 0 }; + let content = &text[content_start..]; + let sections = self.extract_sections(content, frontmatter_lines); + + let mut results = Vec::new(); + + if frontmatter.title.is_some() || !frontmatter.tags.is_empty() { + let frontmatter_text = self.format_frontmatter_chunk(&frontmatter); + if !frontmatter_text.is_empty() { + results.push(SplitResult { + file_path: path.clone(), + window_text: frontmatter_text.clone(), + window_text_hash: official_text_hashing_function(&frontmatter_text), + start_line: 0, + end_line: frontmatter_lines.saturating_sub(1) as u64, + symbol_path: "frontmatter".to_string(), + }); + } + } + + for section in sections { + results.extend(self.chunk_section(§ion, &path)); + } + Ok(results) + } + + fn format_frontmatter_chunk(&self, fm: &KnowledgeFrontmatter) -> String { + let mut parts = Vec::new(); + if let Some(title) = &fm.title { + parts.push(format!("Title: {}", title)); + } + if !fm.tags.is_empty() { + parts.push(format!("Tags: {}", fm.tags.join(", "))); + } + if let Some(kind) = &fm.kind { + parts.push(format!("Kind: {}", kind)); + } + if !fm.filenames.is_empty() { + parts.push(format!("Files: {}", fm.filenames.join(", "))); + } + parts.join("\n") + } + + fn extract_sections(&self, content: &str, line_offset: usize) -> Vec { + let heading_re = Regex::new(r"^(#{1,6})\s+(.+)$").unwrap(); + let code_fence_re = Regex::new(r"^```").unwrap(); + let mut sections = Vec::new(); + let mut current_heading_path: Vec = Vec::new(); + let mut current_content = String::new(); + let mut current_start_line = line_offset; + let mut in_code_block = false; + let lines: Vec<&str> = content.lines().collect(); + + for (idx, line) in lines.iter().enumerate() { + let absolute_line = line_offset + idx; + + if code_fence_re.is_match(line) { + in_code_block = !in_code_block; + current_content.push_str(line); + current_content.push('\n'); + continue; + } + + if in_code_block { + current_content.push_str(line); + current_content.push('\n'); + continue; + } + + if let Some(caps) = heading_re.captures(line) { + if !current_content.trim().is_empty() { + sections.push(MarkdownSection { + heading_path: current_heading_path.clone(), + content: current_content.trim().to_string(), + start_line: current_start_line, + end_line: absolute_line.saturating_sub(1), + }); + } + + let level = caps.get(1).unwrap().as_str().len(); + let heading_text = caps.get(2).unwrap().as_str().to_string(); + while current_heading_path.len() >= level { + current_heading_path.pop(); + } + current_heading_path.push(format!("{} {}", "#".repeat(level), heading_text)); + current_content = format!("{}\n", line); + current_start_line = absolute_line; + } else { + current_content.push_str(line); + current_content.push('\n'); + } + } + + if !current_content.trim().is_empty() { + sections.push(MarkdownSection { + heading_path: current_heading_path, + content: current_content.trim().to_string(), + start_line: current_start_line, + end_line: line_offset + lines.len().saturating_sub(1), + }); + } + sections + } + + fn chunk_section(&self, section: &MarkdownSection, file_path: &PathBuf) -> Vec { + let estimated_tokens = section.content.len() / 4; + + if estimated_tokens <= self.max_tokens { + return vec![SplitResult { + file_path: file_path.clone(), + window_text: section.content.clone(), + window_text_hash: official_text_hashing_function(§ion.content), + start_line: section.start_line as u64, + end_line: section.end_line as u64, + symbol_path: section.heading_path.join(" > "), + }]; + } + + self.split_large_content(§ion.content, section.start_line) + .into_iter() + .map(|(chunk_text, start, end)| SplitResult { + file_path: file_path.clone(), + window_text: chunk_text.clone(), + window_text_hash: official_text_hashing_function(&chunk_text), + start_line: start as u64, + end_line: end as u64, + symbol_path: section.heading_path.join(" > "), + }) + .collect() + } + + fn split_large_content(&self, content: &str, start_line: usize) -> Vec<(String, usize, usize)> { + let mut chunks = Vec::new(); + let lines: Vec<&str> = content.lines().collect(); + let chars_per_chunk = self.max_tokens * 4; + let mut current_chunk = String::new(); + let mut chunk_start = start_line; + let mut current_line = start_line; + + for (idx, line) in lines.iter().enumerate() { + if current_chunk.len() + line.len() + 1 > chars_per_chunk && !current_chunk.is_empty() { + chunks.push((current_chunk.trim().to_string(), chunk_start, current_line.saturating_sub(1))); + let overlap_start = idx.saturating_sub(self.overlap_lines); + current_chunk = lines[overlap_start..idx].join("\n"); + if !current_chunk.is_empty() { + current_chunk.push('\n'); + } + chunk_start = start_line + overlap_start; + } + current_chunk.push_str(line); + current_chunk.push('\n'); + current_line = start_line + idx; + } + + if !current_chunk.trim().is_empty() { + chunks.push((current_chunk.trim().to_string(), chunk_start, start_line + lines.len().saturating_sub(1))); + } + chunks + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_frontmatter_parsing() { + let content = r#"--- +title: "Test Document" +created: 2024-12-17 +tags: ["rust", "testing"] +filenames: ["src/main.rs"] +kind: code +--- + +# Hello World +"#; + let (fm, offset) = KnowledgeFrontmatter::parse(content); + assert_eq!(fm.title, Some("Test Document".to_string())); + assert_eq!(fm.tags, vec!["rust", "testing"]); + assert_eq!(fm.created, Some("2024-12-17".to_string())); + assert_eq!(fm.filenames, vec!["src/main.rs"]); + assert_eq!(fm.kind, Some("code".to_string())); + assert!(offset > 0); + } + + #[test] + fn test_frontmatter_no_frontmatter() { + let content = "# Just a heading\n\nSome content"; + let (fm, offset) = KnowledgeFrontmatter::parse(content); + assert!(fm.title.is_none()); + assert!(fm.tags.is_empty()); + assert_eq!(offset, 0); + } +} diff --git a/refact-agent/engine/src/vecdb/vdb_thread.rs b/refact-agent/engine/src/vecdb/vdb_thread.rs index eebfea887..54ef2a245 100644 --- a/refact-agent/engine/src/vecdb/vdb_thread.rs +++ b/refact-agent/engine/src/vecdb/vdb_thread.rs @@ -14,6 +14,7 @@ use crate::ast::file_splitter::AstBasedFileSplitter; use crate::fetch_embedding::get_embedding_with_retries; use crate::files_in_workspace::{is_path_to_enqueue_valid, Document}; use crate::global_context::GlobalContext; +use crate::vecdb::vdb_markdown_splitter::MarkdownFileSplitter; use crate::vecdb::vdb_sqlite::VecDBSqlite; use crate::vecdb::vdb_structs::{SimpleTextHashVector, SplitResult, VecDbStatus, VecdbConstants, VecdbRecord}; @@ -325,11 +326,24 @@ async fn vectorize_thread( continue; } - let file_splitter = AstBasedFileSplitter::new(constants.splitter_window_size); - let mut splits = file_splitter.vectorization_split(&doc, None, gcx.clone(), constants.embedding_model.base.n_ctx).await.unwrap_or_else(|err| { - info!("{}", err); - vec![] - }); + let is_markdown = doc.doc_path.extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .map(|e| e == "md" || e == "mdx") + .unwrap_or(false); + + let mut splits = if is_markdown { + let md_splitter = MarkdownFileSplitter::new(constants.embedding_model.base.n_ctx); + md_splitter.split(&doc, gcx.clone()).await.unwrap_or_else(|err| { + info!("{}", err); + vec![] + }) + } else { + let file_splitter = AstBasedFileSplitter::new(constants.splitter_window_size); + file_splitter.vectorization_split(&doc, None, gcx.clone(), constants.embedding_model.base.n_ctx).await.unwrap_or_else(|err| { + info!("{}", err); + vec![] + }) + }; // Adding the filename so it can also be searched if let Some(filename) = doc.doc_path.file_name().map(|f| f.to_string_lossy().to_string()) { diff --git a/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml b/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml index f5c17c660..4559e2727 100644 --- a/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml +++ b/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml @@ -55,17 +55,23 @@ SHELL_INSTRUCTIONS: | KNOWLEDGE_INSTRUCTIONS_META: | - Before any action, try to gather existing knowledge: - - Call the `knowledge()` tool to get initial information about the project and the task. - - This tool gives you access to memories, and external data, example trajectories (🗃️) to help understand and solve the task. - Always Learn and Record. Use `create_knowledge()` to: - - Important coding patterns, - - Key decisions and their reasons, - - Effective strategies, - - Insights about the project's structure and dependencies, - - When the task is finished to record useful insights. - - Take every opportunity to build and enrich your knowledge base—don’t wait for instructions. + **KNOWLEDGE MANAGEMENT** + Use the knowledge tools alongside other search tools to build and leverage project-specific knowledge: + **Reading Knowledge** - Use `knowledge(search_key)` to: + - Search for existing documentation, patterns, and decisions before starting work + - Find previous solutions to similar problems + - Understand project conventions and architectural decisions + - Retrieve saved trajectories and insights from past sessions + + **Writing Knowledge** - Use `create_knowledge(tags, content)` to save: + - Important coding patterns discovered during work + - Key architectural decisions and their rationale + - Effective strategies that worked well + - Insights about the project's structure and dependencies + - Solutions to tricky problems for future reference + + Build knowledge continuously - don't wait for explicit instructions to save useful insights. PROMPT_EXPLORATION_TOOLS: | [mode2] You are Refact Chat, a coding assistant. @@ -94,6 +100,8 @@ PROMPT_EXPLORATION_TOOLS: | %GIT_INFO% + %KNOWLEDGE_INSTRUCTIONS% + %PROJECT_TREE% @@ -331,6 +339,7 @@ subchat_tool_parameters: subchat_max_new_tokens: 128000 subchat_reasoning_effort: "high" create_memory_bank: + subchat_model_type: "light" subchat_tokens_for_rag: 120000 subchat_n_ctx: 200000 subchat_max_new_tokens: 10000