From 527522c680b5e567798d13c9a6ac7a7362d64f0b Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Tue, 16 Dec 2025 23:05:34 +1030 Subject: [PATCH 1/8] fix: invalidate memory_document_map cache after file tool operations This fixes a bug where cat() would return stale content after file modifications because the IDE buffer cache (memory_document_map) was not being invalidated when tools wrote to disk. Changes: - auxiliary.rs: Invalidate cache after fs::write() in write_file() (covers create_textdoc, update_textdoc, update_textdoc_by_lines, update_textdoc_regex) - tool_rm.rs: Invalidate cache after file/directory deletion (with prefix removal for directories) - tool_mv.rs: Invalidate cache for source and destination paths after rename, overwrite, and cross-device copy operations Root cause: memory_document_map is an LSP cache for files open in IDE. Tools were writing to disk but not invalidating this cache, causing subsequent cat() calls to return stale IDE buffer content instead of fresh disk content. --- .../engine/src/agentic/generate_code_edit.rs | 163 ++++++++++++++++++ refact-agent/engine/src/agentic/mod.rs | 3 +- refact-agent/engine/src/http/routers/v1.rs | 3 + .../engine/src/http/routers/v1/code_edit.rs | 52 ++++++ .../engine/src/tools/file_edit/auxiliary.rs | 2 + refact-agent/engine/src/tools/tool_mv.rs | 26 +++ refact-agent/engine/src/tools/tool_rm.rs | 14 ++ 7 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 refact-agent/engine/src/agentic/generate_code_edit.rs create mode 100644 refact-agent/engine/src/http/routers/v1/code_edit.rs diff --git a/refact-agent/engine/src/agentic/generate_code_edit.rs b/refact-agent/engine/src/agentic/generate_code_edit.rs new file mode 100644 index 000000000..7c2cf7ffd --- /dev/null +++ b/refact-agent/engine/src/agentic/generate_code_edit.rs @@ -0,0 +1,163 @@ +use crate::at_commands::at_commands::AtCommandsContext; +use crate::call_validation::{ChatContent, ChatMessage}; +use crate::global_context::{try_load_caps_quickly_if_not_present, GlobalContext}; +use crate::subchat::subchat_single; +use std::sync::Arc; +use tokio::sync::Mutex as AMutex; +use tokio::sync::RwLock as ARwLock; + +const CODE_EDIT_SYSTEM_PROMPT: &str = r#"You are a code editing assistant. Your task is to modify the provided code according to the user's instruction. + +# Rules +1. Return ONLY the edited code - no explanations, no markdown fences, no commentary +2. Preserve the original indentation style and formatting conventions +3. Make minimal changes necessary to fulfill the instruction +4. If the instruction is unclear, make the most reasonable interpretation +5. Keep all code that isn't directly related to the instruction unchanged + +# Output Format +Return the edited code directly, without any wrapping or explanation. The output should be valid code that can directly replace the input."#; + +const N_CTX: usize = 32000; +const TEMPERATURE: f32 = 0.1; + +fn remove_markdown_fences(text: &str) -> String { + let trimmed = text.trim(); + if trimmed.starts_with("```") { + let lines: Vec<&str> = trimmed.lines().collect(); + if lines.len() >= 2 { + // Find closing fence + if let Some(end_idx) = lines.iter().rposition(|l| l.trim() == "```") { + if end_idx > 0 { + // Skip first line (```language) and last line (```) + let start_idx = 1; + if start_idx < end_idx { + return lines[start_idx..end_idx].join("\n"); + } + } + } + } + } + text.to_string() +} + +pub async fn generate_code_edit( + gcx: Arc>, + code: &str, + instruction: &str, + cursor_file: &str, + cursor_line: i32, +) -> Result { + if code.is_empty() { + return Err("The provided code is empty".to_string()); + } + if instruction.is_empty() { + return Err("The instruction is empty".to_string()); + } + + let user_message = format!( + "File: {} (line {})\n\nCode to edit:\n```\n{}\n```\n\nInstruction: {}", + cursor_file, cursor_line, code, instruction + ); + + let messages = vec![ + ChatMessage { + role: "system".to_string(), + content: ChatContent::SimpleText(CODE_EDIT_SYSTEM_PROMPT.to_string()), + ..Default::default() + }, + ChatMessage { + role: "user".to_string(), + content: ChatContent::SimpleText(user_message), + ..Default::default() + }, + ]; + + let model_id = match try_load_caps_quickly_if_not_present(gcx.clone(), 0).await { + Ok(caps) => { + // Prefer light model for fast inline edits, fallback to default + let light = &caps.defaults.chat_light_model; + if !light.is_empty() { + Ok(light.clone()) + } else { + Ok(caps.defaults.chat_default_model.clone()) + } + } + Err(_) => Err("No caps available".to_string()), + }?; + + let ccx: Arc> = Arc::new(AMutex::new( + AtCommandsContext::new( + gcx.clone(), + N_CTX, + 1, + false, + messages.clone(), + "".to_string(), + false, + model_id.clone(), + ) + .await, + )); + + let new_messages = subchat_single( + ccx.clone(), + &model_id, + messages, + Some(vec![]), // No tools - pure generation + None, + false, + Some(TEMPERATURE), + None, + 1, + None, + false, // Don't prepend system prompt - we have our own + None, + None, + None, + ) + .await + .map_err(|e| format!("Error generating code edit: {}", e))?; + + let edited_code = new_messages + .into_iter() + .next() + .and_then(|msgs| msgs.into_iter().last()) + .and_then(|msg| match msg.content { + ChatContent::SimpleText(text) => Some(text), + ChatContent::Multimodal(_) => None, + }) + .ok_or("No edited code was generated".to_string())?; + + // Strip markdown fences if present + Ok(remove_markdown_fences(&edited_code)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_remove_markdown_fences_with_language() { + let input = "```python\ndef hello():\n print('world')\n```"; + assert_eq!(remove_markdown_fences(input), "def hello():\n print('world')"); + } + + #[test] + fn test_remove_markdown_fences_without_language() { + let input = "```\nsome code\n```"; + assert_eq!(remove_markdown_fences(input), "some code"); + } + + #[test] + fn test_remove_markdown_fences_no_fences() { + let input = "plain code without fences"; + assert_eq!(remove_markdown_fences(input), "plain code without fences"); + } + + #[test] + fn test_remove_markdown_fences_with_whitespace() { + let input = " ```rust\nfn main() {}\n``` "; + assert_eq!(remove_markdown_fences(input), "fn main() {}"); + } +} diff --git a/refact-agent/engine/src/agentic/mod.rs b/refact-agent/engine/src/agentic/mod.rs index 6a05dbfa3..6c3f144b9 100644 --- a/refact-agent/engine/src/agentic/mod.rs +++ b/refact-agent/engine/src/agentic/mod.rs @@ -1,3 +1,4 @@ pub mod generate_commit_message; pub mod generate_follow_up_message; -pub mod compress_trajectory; \ No newline at end of file +pub mod compress_trajectory; +pub mod generate_code_edit; \ No newline at end of file diff --git a/refact-agent/engine/src/http/routers/v1.rs b/refact-agent/engine/src/http/routers/v1.rs index 8e3c86a8a..60669b16e 100644 --- a/refact-agent/engine/src/http/routers/v1.rs +++ b/refact-agent/engine/src/http/routers/v1.rs @@ -37,6 +37,7 @@ use crate::http::routers::v1::providers::{handle_v1_providers, handle_v1_provide use crate::http::routers::v1::vecdb::{handle_v1_vecdb_search, handle_v1_vecdb_status}; use crate::http::routers::v1::v1_integrations::{handle_v1_integration_get, handle_v1_integration_icon, handle_v1_integration_save, handle_v1_integration_delete, handle_v1_integrations, handle_v1_integrations_filtered, handle_v1_integrations_mcp_logs}; use crate::http::routers::v1::file_edit_tools::handle_v1_file_edit_tool_dry_run; +use crate::http::routers::v1::code_edit::handle_v1_code_edit; use crate::http::routers::v1::workspace::{handle_v1_get_app_searchable_id, handle_v1_set_active_group_id}; mod ast; @@ -64,6 +65,7 @@ pub mod telemetry_chat; pub mod telemetry_network; pub mod providers; mod file_edit_tools; +mod code_edit; mod v1_integrations; pub mod vecdb; mod workspace; @@ -135,6 +137,7 @@ pub fn make_v1_router() -> Router { .route("/links", post(handle_v1_links)) .route("/file_edit_tool_dry_run", post(handle_v1_file_edit_tool_dry_run)) + .route("/code-edit", post(handle_v1_code_edit)) .route("/providers", get(handle_v1_providers)) .route("/provider-templates", get(handle_v1_provider_templates)) diff --git a/refact-agent/engine/src/http/routers/v1/code_edit.rs b/refact-agent/engine/src/http/routers/v1/code_edit.rs new file mode 100644 index 000000000..a4495ed71 --- /dev/null +++ b/refact-agent/engine/src/http/routers/v1/code_edit.rs @@ -0,0 +1,52 @@ +use crate::agentic::generate_code_edit::generate_code_edit; +use crate::custom_error::ScratchError; +use crate::global_context::GlobalContext; +use axum::http::{Response, StatusCode}; +use axum::Extension; +use hyper::Body; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::RwLock as ARwLock; + +#[derive(Deserialize)] +pub struct CodeEditPost { + pub code: String, + pub instruction: String, + pub cursor_file: String, + pub cursor_line: i32, +} + +#[derive(Serialize)] +pub struct CodeEditResponse { + pub edited_code: String, +} + +pub async fn handle_v1_code_edit( + Extension(global_context): Extension>>, + body_bytes: hyper::body::Bytes, +) -> axum::response::Result, ScratchError> { + let post = serde_json::from_slice::(&body_bytes).map_err(|e| { + ScratchError::new( + StatusCode::UNPROCESSABLE_ENTITY, + format!("JSON problem: {}", e), + ) + })?; + + let edited_code = generate_code_edit( + global_context.clone(), + &post.code, + &post.instruction, + &post.cursor_file, + post.cursor_line, + ) + .await + .map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, e))?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::to_string(&CodeEditResponse { edited_code }).unwrap(), + )) + .unwrap()) +} diff --git a/refact-agent/engine/src/tools/file_edit/auxiliary.rs b/refact-agent/engine/src/tools/file_edit/auxiliary.rs index a7036182f..d72b400be 100644 --- a/refact-agent/engine/src/tools/file_edit/auxiliary.rs +++ b/refact-agent/engine/src/tools/file_edit/auxiliary.rs @@ -169,6 +169,8 @@ pub async fn write_file(gcx: Arc>, path: &PathBuf, file_t warn!("{err}"); err })?; + // Invalidate stale cache entry so subsequent reads get fresh content from disk + gcx.write().await.documents_state.memory_document_map.remove(path); } Ok((before_text, file_text.to_string())) diff --git a/refact-agent/engine/src/tools/tool_mv.rs b/refact-agent/engine/src/tools/tool_mv.rs index 2e4d4dd90..2c195b50c 100644 --- a/refact-agent/engine/src/tools/tool_mv.rs +++ b/refact-agent/engine/src/tools/tool_mv.rs @@ -156,12 +156,26 @@ impl Tool for ToolMv { if dst_metadata.is_dir() { fs::remove_dir_all(&dst_true_path).await .map_err(|e| format!("Failed to remove existing directory '{}': {}", dst_str, e))?; + // Invalidate cache entries for all files under the removed directory + { + let mut gcx_write = gcx.write().await; + let paths_to_remove: Vec<_> = gcx_write.documents_state.memory_document_map + .keys() + .filter(|p| p.starts_with(&dst_true_path)) + .cloned() + .collect(); + for p in paths_to_remove { + gcx_write.documents_state.memory_document_map.remove(&p); + } + } } else { if !dst_metadata.is_dir() { dst_file_content = fs::read_to_string(&dst_true_path).await.unwrap_or_else(|_| "".to_string()); } fs::remove_file(&dst_true_path).await .map_err(|e| format!("Failed to remove existing file '{}': {}", dst_str, e))?; + // Invalidate cache entry for the removed file + gcx.write().await.documents_state.memory_document_map.remove(&dst_true_path); } } @@ -179,6 +193,12 @@ impl Tool for ToolMv { match fs::rename(&src_true_path, &dst_true_path).await { Ok(_) => { + // Invalidate cache entries for both source and destination + { + let mut gcx_write = gcx.write().await; + gcx_write.documents_state.memory_document_map.remove(&src_true_path); + gcx_write.documents_state.memory_document_map.remove(&dst_true_path); + } let corrections = src_str != src_corrected_path || dst_str != dst_corrected_path; let mut messages = vec![]; if !src_is_dir && !src_file_content.is_empty() { @@ -235,6 +255,12 @@ impl Tool for ToolMv { .map_err(|e| format!("Failed to copy '{}' to '{}': {}", src_str, dst_str, e))?; fs::remove_file(&src_true_path).await .map_err(|e| format!("Failed to remove source file '{}' after copy: {}", src_str, e))?; + // Invalidate cache entries for both source and destination + { + let mut gcx_write = gcx.write().await; + gcx_write.documents_state.memory_document_map.remove(&src_true_path); + gcx_write.documents_state.memory_document_map.remove(&dst_true_path); + } let mut messages = vec![]; diff --git a/refact-agent/engine/src/tools/tool_rm.rs b/refact-agent/engine/src/tools/tool_rm.rs index 7e15eb66a..522b97c2d 100644 --- a/refact-agent/engine/src/tools/tool_rm.rs +++ b/refact-agent/engine/src/tools/tool_rm.rs @@ -203,6 +203,18 @@ impl Tool for ToolRm { fs::remove_dir_all(&true_path).await.map_err(|e| { format!("Failed to remove directory '{}': {}", corrected_path, e) })?; + // Invalidate cache entries for all files under the deleted directory + { + let mut gcx_write = gcx.write().await; + let paths_to_remove: Vec<_> = gcx_write.documents_state.memory_document_map + .keys() + .filter(|p| p.starts_with(&true_path)) + .cloned() + .collect(); + for p in paths_to_remove { + gcx_write.documents_state.memory_document_map.remove(&p); + } + } messages.push(ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), content: ChatContent::SimpleText(format!("Removed directory '{}'", corrected_path)), @@ -224,6 +236,8 @@ impl Tool for ToolRm { fs::remove_file(&true_path).await.map_err(|e| { format!("Failed to remove file '{}': {}", corrected_path, e) })?; + // Invalidate cache entry for the deleted file + gcx.write().await.documents_state.memory_document_map.remove(&true_path); if !file_content.is_empty() { let diff_chunk = DiffChunk { file_name: corrected_path.clone(), From 8c1a7ac33802b48cd635553d733cf38a4af392a7 Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 04:46:55 +1030 Subject: [PATCH 2/8] feat: Local markdown-based knowledge system - Replace cloud GraphQL knowledge backend with local markdown files - Store knowledge in .refact_knowledge/ folder with YAML frontmatter - Add markdown-aware file splitter for VecDB indexing - Add create_agents_md tool to generate project guide - Add filenames parameter to create_knowledge tool - Save trajectories to .refact_knowledge/trajectories/ - Update prompts with knowledge management instructions - Remove cloud module and related code completely - Allowlist .refact_knowledge/ folder for indexing despite being hidden New tools: - knowledge(search_key) - search local knowledge - create_knowledge(tags, filenames, content) - create knowledge entry - create_agents_md() - generate AGENTS.md from all knowledge Files changed: - New: src/vecdb/vdb_markdown_splitter.rs - New: src/tools/tool_create_agents_md.rs - Rewritten: src/memories.rs - Deleted: src/cloud/ directory - Updated: prompts, tools, file filtering --- ...42754_refact-agent-project-architecture.md | 149 +++++ ...7_043144_core-architecture-entry-points.md | 415 ++++++++++++++ ...7_044219_c-treesitter-parser-test-cases.md | 54 ++ ...44254_java-treesitter-parser-test-cases.md | 54 ++ ...javascript-treesitter-parser-test-cases.md | 43 ++ .../2025-12-17_044355_knowledge.md | 49 ++ ...418_python-treesitter-parser-test-cases.md | 36 ++ ...44438_rust-treesitter-parser-test-cases.md | 41 ++ ...typescript-treesitter-parser-test-cases.md | 39 ++ ..._treesitter-parser-test-cases-directory.md | 53 ++ ...tagentguisrcfeaturesprovidersproviderfo.md | 53 ++ ...tagentguisrcfeaturesprovidersproviderfo.md | 53 ++ .../2025-12-17_044542_analysis.md | 140 +++++ AGENTS.md | 0 .../engine/src/at_commands/at_knowledge.rs | 43 +- refact-agent/engine/src/background_tasks.rs | 6 +- refact-agent/engine/src/cloud/experts_req.rs | 130 ----- refact-agent/engine/src/cloud/messages_req.rs | 365 ------------- refact-agent/engine/src/cloud/mod.rs | 4 - refact-agent/engine/src/cloud/threads_req.rs | 300 ---------- refact-agent/engine/src/cloud/threads_sub.rs | 512 ------------------ refact-agent/engine/src/constants.rs | 2 - refact-agent/engine/src/file_filter.rs | 16 +- refact-agent/engine/src/files_in_workspace.rs | 1 - refact-agent/engine/src/global_context.rs | 4 - .../http/routers/v1/chat_based_handlers.rs | 18 +- .../engine/src/http/routers/v1/workspace.rs | 3 +- refact-agent/engine/src/main.rs | 1 - refact-agent/engine/src/memories.rs | 471 +++++++--------- .../src/scratchpads/chat_utils_prompts.rs | 20 +- .../engine/src/tools/tool_create_knowledge.rs | 76 ++- .../src/tools/tool_create_memory_bank.rs | 6 +- .../engine/src/tools/tool_knowledge.rs | 64 ++- refact-agent/engine/src/tools/tools_list.rs | 14 +- refact-agent/engine/src/vecdb/mod.rs | 1 + .../engine/src/vecdb/vdb_markdown_splitter.rs | 272 ++++++++++ refact-agent/engine/src/vecdb/vdb_thread.rs | 24 +- .../customization_compiled_in.yaml | 29 +- 38 files changed, 1841 insertions(+), 1720 deletions(-) create mode 100644 .refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md create mode 100644 .refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md create mode 100644 .refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md create mode 100644 .refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md create mode 100644 .refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md create mode 100644 .refact_knowledge/2025-12-17_044355_knowledge.md create mode 100644 .refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md create mode 100644 .refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md create mode 100644 .refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md create mode 100644 .refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md create mode 100644 .refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md create mode 100644 .refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md create mode 100644 .refact_knowledge/trajectories/2025-12-17_044542_analysis.md create mode 100644 AGENTS.md delete mode 100644 refact-agent/engine/src/cloud/experts_req.rs delete mode 100644 refact-agent/engine/src/cloud/messages_req.rs delete mode 100644 refact-agent/engine/src/cloud/mod.rs delete mode 100644 refact-agent/engine/src/cloud/threads_req.rs delete mode 100644 refact-agent/engine/src/cloud/threads_sub.rs create mode 100644 refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs diff --git a/.refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md b/.refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md new file mode 100644 index 000000000..8a48efad8 --- /dev/null +++ b/.refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md @@ -0,0 +1,149 @@ +--- +title: "Refact Agent Project Architecture" +created: 2025-12-17 +tags: ["architecture", "project-structure", "rust", "lsp", "refact-agent"] +--- + +## Refact Agent Project Architecture + +### Project Overview +Refact Agent is a Rust-based LSP (Language Server Protocol) server designed to integrate with IDEs like VSCode and JetBrains. It maintains up-to-date AST (Abstract Syntax Tree) and VecDB (Vector Database) indexes for efficient code completion and project analysis. + +### Core Components + +#### 1. **refact-agent/engine** (Main Rust Application) +The heart of the project, containing: +- **src/main.rs** - Entry point for the LSP server +- **src/lsp.rs** - LSP protocol implementation +- **src/http.rs** - HTTP server for IDE communication +- **src/background_tasks.rs** - Async background task management +- **src/global_context.rs** - Shared application state + +#### 2. **Key Modules** + +**AST Module** (`src/ast/`) +- Handles Abstract Syntax Tree parsing and indexing +- Supports multiple programming languages +- Provides symbol definitions and references + +**VecDB Module** (`src/vecdb/`) +- Vector database for semantic code search +- Markdown splitting and indexing +- Efficient similarity-based code retrieval + +**AT Commands** (`src/at_commands/`) +- Special commands for IDE integration (e.g., @knowledge, @file) +- Command parsing and execution +- Context-aware completions + +**Tools** (`src/tools/`) +- External tool integrations (browsers, databases, debuggers) +- Tool authorization and execution +- Tool result processing + +**Integrations** (`src/integrations/`) +- Third-party service integrations +- API clients and handlers + +**HTTP Module** (`src/http/`) +- REST API endpoints for IDE clients +- Request/response handling +- Streaming support for long-running operations + +#### 3. **Supporting Systems** + +**Completion Cache** (`src/completion_cache.rs`) +- Caches completion results for performance +- Invalidation strategies + +**File Management** (`src/files_*.rs`) +- File filtering and blocklisting +- Workspace file discovery +- File correction and caching + +**Telemetry** (`src/telemetry/`) +- Usage tracking and analytics +- Privacy-aware data collection + +**Postprocessing** (`src/postprocessing/`) +- Result formatting and enhancement +- Output optimization + +### Project Structure + +``` +refact-agent/ +├── engine/ +│ ├── src/ +│ │ ├── agentic/ # Agentic features +│ │ ├── ast/ # AST parsing and indexing +│ │ ├── at_commands/ # IDE command handlers +│ │ ├── caps/ # Capabilities management +│ │ ├── cloud/ # Cloud integration +│ │ ├── dashboard/ # Dashboard features +│ │ ├── git/ # Git integration +│ │ ├── http/ # HTTP server +│ │ ├── integrations/ # External integrations +│ │ ├── postprocessing/ # Result processing +│ │ ├── scratchpads/ # Scratchpad management +│ │ ├── telemetry/ # Analytics +│ │ ├── tools/ # Tool integrations +│ │ ├── vecdb/ # Vector database +│ │ ├── yaml_configs/ # Configuration handling +│ │ └── main.rs, lsp.rs, http.rs, etc. +│ ├── tests/ # Python test scripts +│ ├── examples/ # Usage examples +│ └── Cargo.toml +├── gui/ # TypeScript/React frontend +└── python_binding_and_cmdline/ # Python bindings +``` + +### Key Technologies + +- **Language**: Rust (backend), TypeScript/React (frontend) +- **Protocol**: LSP (Language Server Protocol) +- **Database**: SQLite with vector extensions +- **APIs**: REST, GraphQL +- **Testing**: Python test scripts, Rust unit tests + +### Development Workflow + +1. **Building**: `cargo build` in `refact-agent/engine` +2. **Testing**: Python test scripts in `tests/` directory +3. **Running**: LSP server runs as background process +4. **Configuration**: YAML-based configuration system + +### Important Files + +- `Cargo.toml` - Rust dependencies and workspace configuration +- `known_models.json` - Supported AI models configuration +- `build.rs` - Build script for code generation +- `rustfmt.toml` - Code formatting rules + +### Integration Points + +- **IDEs**: VSCode, JetBrains (via LSP) +- **External APIs**: OpenAI, Hugging Face, custom endpoints +- **Databases**: SQLite, Vector databases +- **Version Control**: Git integration +- **Cloud Services**: Cloud-based features and authentication + +### Current Branch +- **Active**: `debug_fixes_pt_42` +- **Main branches**: `main`, `dev`, `cloud-subchats` +- **Staging**: 2 files, Modified: 23 files + +### Testing Infrastructure + +- **Unit Tests**: Rust tests in source files +- **Integration Tests**: Python scripts in `tests/` directory +- **Examples**: Executable examples in `examples/` directory +- **Test Data**: Sample data in `tests/test13_data/` + +### Performance Considerations + +- Completion caching for reduced latency +- Async background tasks for non-blocking operations +- Vector database for efficient semantic search +- Streaming responses for large outputs + diff --git a/.refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md b/.refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md new file mode 100644 index 000000000..8f0166d00 --- /dev/null +++ b/.refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md @@ -0,0 +1,415 @@ +--- +title: "Core Architecture & Entry Points" +created: 2025-12-17 +tags: ["architecture", "core-modules", "patterns", "design", "rust", "lsp", "refact-agent", "configuration", "testing"] +--- + +## Core Architecture & Entry Points + +### Main Entry Points + +**main.rs** - Application bootstrap +- Initializes `GlobalContext` with all subsystems +- Spawns HTTP server (Axum) and LSP server (tower-lsp) +- Handles graceful shutdown and signal management +- Loads configuration from YAML files + +**lsp.rs** - Language Server Protocol implementation +- Implements tower-lsp traits for IDE communication +- Handles document synchronization +- Manages workspace symbols and definitions +- Bridges LSP requests to internal services + +**http.rs** - REST API server +- Axum-based HTTP server for IDE clients +- Endpoints for completion, chat, RAG, tools +- Streaming response support +- Request validation and error handling + +### GlobalContext - The Central Hub + +Located in `global_context.rs`, this is the "god object" that coordinates all subsystems: + +**Key Responsibilities:** +- Shared mutable state (Arc>) +- AST indexing service +- VecDB (vector database) management +- File watching and caching +- Model provider configuration +- Tool execution context +- Telemetry and analytics + +**Access Pattern:** +``` +HTTP/LSP Request → GlobalContext.read() → Service Layer → Response + → GlobalContext.write() → State Update +``` + +**Important Fields:** +- `ast_service` - AST indexing and symbol resolution +- `vecdb` - Vector database for semantic search +- `file_cache` - Completion and file caching +- `caps` - Model capabilities and providers +- `tool_executor` - Tool execution engine +- `background_tasks` - Async task management + +### Dual Protocol Architecture + +**HTTP Server (Axum)** +- Primary interface for IDE clients +- RESTful endpoints +- Streaming support for long operations +- CORS and authentication handling + +**LSP Server (tower-lsp)** +- Secondary interface for IDE integration +- Document synchronization +- Workspace symbol queries +- Hover, definition, references + +**Shared State:** +Both servers access the same `GlobalContext`, ensuring consistency across protocols. + +--- + +## Core Modules + +### AST Module (`src/ast/`) + +**Purpose:** Parse and index code structure across multiple languages + +**Key Components:** +- `AstIndexService` - Main indexing service +- Tree-sitter integration for 6+ languages (Rust, Python, JavaScript, TypeScript, Go, Java) +- Symbol definition and reference tracking +- Incremental indexing on file changes + +**Key Functions:** +- `ast_definition()` - Find symbol definition +- `ast_references()` - Find all symbol usages +- `pick_up_changes()` - Incremental indexing + +**Design Pattern:** +- Background task updates AST on file changes +- Caches results for performance +- Fallback to file content if AST unavailable + +### VecDB Module (`src/vecdb/`) + +**Purpose:** Vector database for semantic code search and RAG + +**Key Components:** +- `VecDb` - Main vector database interface +- SQLite backend with vector extensions +- Markdown splitter for code chunking +- Embedding generation and storage + +**Key Functions:** +- `search()` - Semantic similarity search +- `index()` - Add code to vector database +- `get_status()` - VecDB indexing status + +**Design Pattern:** +- Lazy initialization on first use +- Background indexing of workspace files +- Fallback to keyword search if vectors unavailable +- Configurable embedding models + +### AT Commands Module (`src/at_commands/`) + +**Purpose:** Special IDE commands for context injection and tool execution + +**Key Commands:** +- `@file` - Include file content +- `@knowledge` - Search knowledge base +- `@definition` - Find symbol definition +- `@references` - Find symbol usages +- `@web` - Web search integration +- `@tool` - Execute external tools + +**Design Pattern:** +- Command parsing and validation +- Context gathering from various sources +- Result formatting for LLM consumption +- Authorization checks for sensitive operations + +### Tools Module (`src/tools/`) + +**Purpose:** Execute external tools and integrate with external services + +**Key Tools:** +- `tool_ast_definition` - AST-based symbol lookup +- `tool_create_agents_md` - Generate project documentation +- `tool_web_search` - Web search integration +- `tool_execute_command` - Shell command execution +- Custom tool support via configuration + +**Design Pattern:** +- Tool registry with metadata +- Authorization and permission checking +- Result validation and sanitization +- Error handling and fallbacks + +### Integrations Module (`src/integrations/`) + +**Purpose:** Third-party service integrations + +**Key Integrations:** +- OpenAI API client +- Hugging Face integration +- Custom LLM endpoints +- Cloud service connections + +**Design Pattern:** +- Provider abstraction layer +- Fallback chains for redundancy +- Rate limiting and caching +- Error recovery strategies + +--- + +## Configuration System + +### YAML-Driven Configuration + +**Location:** `src/yaml_configs/` + +**Key Features:** +- Auto-generated configuration files +- Checksum validation for integrity +- Hot-reload capability +- Hierarchical configuration + +**Configuration Files:** +- `providers.yaml` - Model provider definitions +- `capabilities.yaml` - Feature capabilities +- `tools.yaml` - Tool definitions and permissions +- `integrations.yaml` - Integration settings + +**Design Pattern:** +- Configuration as code +- Validation on load +- Graceful degradation on missing configs +- Environment variable overrides + +--- + +## Performance & Caching + +### Completion Cache (`src/completion_cache.rs`) + +**Purpose:** Cache completion results for repeated queries + +**Strategy:** +- LRU cache with configurable size +- Invalidation on file changes +- Workspace-aware caching + +### File Correction Cache (`src/files_correction_cache.rs`) + +**Purpose:** Cache file correction results + +**Strategy:** +- Persistent cache with TTL +- Invalidation on file modifications + +### Background Tasks (`src/background_tasks.rs`) + +**Purpose:** Async operations without blocking main thread + +**Key Tasks:** +- AST indexing +- VecDB updates +- File watching +- Telemetry collection + +**Design Pattern:** +- Tokio-based async runtime +- Task prioritization +- Graceful shutdown handling + +--- + +## Testing Infrastructure + +### Test Organization + +**Location:** `refact-agent/engine/tests/` + +**Test Types:** +1. **Integration Tests** - Python scripts testing HTTP/LSP endpoints +2. **Unit Tests** - Rust tests in source files +3. **Examples** - Executable examples in `examples/` + +### Key Test Files + +- `test01_completion_edge_cases.py` - Completion edge cases +- `test02_completion_with_rag.py` - RAG integration +- `test03_at_commands_completion.py` - @command testing +- `test04_completion_lsp.py` - LSP protocol testing +- `test05_is_openai_compatible.py` - OpenAI API compatibility +- `test12_tools_authorize_calls.py` - Tool authorization +- `test13_vision.py` - Vision/image capabilities + +### Test Patterns + +- Python test scripts use HTTP client +- LSP tests use `lsp_connect.py` helper +- Test data in `tests/test13_data/` +- Emergency test data in `tests/emergency_frog_situation/` + +--- + +## Key Design Patterns + +### 1. Layered Fallback Pattern + +``` +Request → Cache → VecDB → AST → Model Inference + (fast) (semantic) (structural) (comprehensive) +``` + +### 2. Background Task Pattern + +``` +Main Thread (HTTP/LSP) ← Shared State → Background Threads (Indexing/Watching) +``` + +### 3. Provider Abstraction + +``` +GlobalContext → Capabilities → Provider Selection → Model Inference +``` + +### 4. Command Execution Pattern + +``` +@command → Parser → Validator → Executor → Formatter → Response +``` + +### 5. Error Recovery Pattern + +``` +Try Primary → Catch Error → Try Fallback → Log & Return Default +``` + +--- + +## Important Utilities + +### File Management (`src/files_*.rs`) + +- `files_in_workspace.rs` - Discover workspace files +- `files_blocklist.rs` - Filter blocked files +- `files_correction.rs` - Fix file paths and content +- `file_filter.rs` - Apply filtering rules + +### Privacy & Security (`src/privacy.rs`) + +- Sensitive data masking +- Privacy-aware logging +- Data sanitization + +### Telemetry (`src/telemetry/`) + +- Usage tracking +- Analytics collection +- Privacy-compliant reporting + +### Utilities + +- `tokens.rs` - Token counting and management +- `json_utils.rs` - JSON parsing helpers +- `fuzzy_search.rs` - Fuzzy matching +- `nicer_logs.rs` - Enhanced logging + +--- + +## Dependencies & Technology Stack + +### Core Dependencies + +- **tower-lsp** - LSP server implementation +- **axum** - HTTP server framework +- **tokio** - Async runtime +- **tree-sitter** - Code parsing (6+ languages) +- **sqlite-vec** - Vector database +- **serde** - Serialization +- **reqwest** - HTTP client + +### Language Support + +- Rust +- Python +- JavaScript/TypeScript +- Go +- Java +- C/C++ + +### External Services + +- OpenAI API +- Hugging Face +- Custom LLM endpoints +- Web search APIs +- Cloud services + +--- + +## Development Workflow + +### Building + +```bash +cd refact-agent/engine +cargo build --release +``` + +### Testing + +```bash +# Run all tests +cargo test --workspace + +# Run specific test +python tests/test01_completion_edge_cases.py +``` + +### Running + +```bash +cargo run --bin refact-lsp +``` + +### Configuration + +- YAML files in `src/yaml_configs/` +- Environment variables for overrides +- Hot-reload on file changes + +--- + +## Current State & Branches + +**Active Branch:** `debug_fixes_pt_42` + +**Main Branches:** +- `main` - Stable release +- `dev` - Development +- `cloud-subchats` - Cloud features +- `main-stable-2` - Previous stable + +**Staged Changes:** 2 files +**Modified Files:** 23 files + +--- + +## Key Insights + +1. **Scalability**: Dual protocol (HTTP + LSP) allows flexible IDE integration +2. **Performance**: Multi-layer caching and background indexing minimize latency +3. **Extensibility**: YAML-driven configuration enables easy feature addition +4. **Reliability**: Fallback chains and error recovery ensure graceful degradation +5. **Maintainability**: Clear separation of concerns across modules +6. **Testing**: Comprehensive integration tests validate functionality + diff --git a/.refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md new file mode 100644 index 000000000..bf65ad05b --- /dev/null +++ b/.refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md @@ -0,0 +1,54 @@ +--- +title: "C++ Tree-sitter Parser Test Cases" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "cpp", "refact-agent"] +--- + +### C++ Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains test fixtures (sample C++ source files and their expected outputs) specifically for validating the C++ Tree-sitter parser implementation within the refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world C++ code for features like "go to definition" (`ast_definition`), "find references" (`ast_references`), and code analysis in the LSP server. The tests build upon the core AST module's incremental indexing and multi-language support by providing language-specific validation data. + +**• Files:** +``` +cpp/ +├── circle.cpp # Sample C++ source: likely tests class/struct definitions, methods +├── circle.cpp.decl_json # Expected JSON output: extracted declarations/symbol table +├── circle.cpp.skeleton # Expected skeletonized version: structure-only (no bodies/implementation) +├── main.cpp # Sample C++ source: entry point, function calls, includes +└── main.cpp.json # Expected JSON output: full AST parse or symbol info +``` +- **Organization pattern**: Each test case pairs a `.cpp` source file with companion `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This mirrors test setups in sibling directories (`java/`, `python/`, etc.), enabling consistent cross-language validation. +- **Naming convention**: `filename.lang[.variant].{json|decl_json|skeleton}` – clear, machine-readable, focused on parser outputs. +- Notable: Directory reported as "empty" in file read, but structure confirms 5 fixture files present for targeted C++ testing. + +**• Architecture:** +- **Role in AST pipeline**: Part of `src/ast/treesitter/parsers/tests/cases/` hierarchy, consumed by `tests/cpp.rs` (test runner module). Tests invoke Tree-sitter's C++ grammar (`parsers/cpp.rs`) to parse files, then validate against golden `.json`/`.skeleton` outputs. +- **Design patterns**: + - **Golden file testing**: Compare runtime parser output vs. pre-approved fixtures for regression-proofing. + - **Modular language isolation**: Per-language subdirs allow independent grammar evolution without affecting others (e.g., Rust/Python parsers unchanged). + - **Multi-output validation**: Tests three concerns simultaneously – raw AST (`json`), symbols (`decl_json`), structure (`skeleton`) – covering the full AST-to-analysis flow. +- **Data flow**: `file_ast_markup.rs` or `skeletonizer.rs` processes `.cpp` → generates AST/symbols → serializes to JSON → `tests/cpp.rs` asserts equality. +- **Fits layered architecture**: Bottom layer (Tree-sitter parsing) → tested here → feeds `ast_db.rs`/`ast_indexer_thread.rs` for background indexing → used by `at_ast_definition.rs`/`at_ast_reference.rs`. +- **Error handling**: Implicit via test failures; likely uses `anyhow` or custom `custom_error.rs` for parse errors. +- **Extension points**: Easy to add new C++ edge cases (templates, lambdas, STL) without code changes. + +**• Key Symbols (inferred from test consumers):** +- From `ast_instance_structs.rs`/`structs.rs`: `AstInstance`, `SymbolDecl`, `SkeletonNode` – parsed/validated here. +- Parser entry: `parsers/cpp.rs::parse_cpp()` or similar – Tree-sitter query capture for C++ nodes. +- Test harness: `tests/cpp.rs` likely exports `test_cpp_cases()` calling `language_id.rs::Cpp`, `file_ast_markup.rs::markup_file()`. +- Cross-references: Relies on `parse_common.rs`, `utils.rs`; outputs feed `ast_structs.rs::Node`. + +**• Integration:** +- **Used by**: `src/ast/treesitter/parsers/tests/cpp.rs` (direct test runner); indirectly powers `at_ast_definition.rs`, `tool_ast_definition.rs`, `ast_indexer_thread.rs` for live IDE queries. +- **Uses from others**: Tree-sitter grammars (`parsers/cpp.rs`), shared utils (`treesitter/utils.rs`, `chunk_utils.rs`), `language_id.rs` for C++ detection. +- **Relationships**: + | Depends On | Used By | Communication | + |---------------------|-----------------------------|---------------| + | `parsers/cpp.rs` | `tests/cpp.rs` | File paths, parse results | + | `skeletonizer.rs` | `ast_db.rs` | Skeleton strings | + | `language_id.rs` | `ast_parse_anything.rs` | Language enum (Cpp) | +- **Comes after**: General `alt_testsuite/` (annotated complex cases like `cpp_goat_library.cpp`); more focused than multi-lang `tests.ts`. +- **Comparison to existing knowledge**: Builds upon AST module's "Tree-sitter integration for 6+ languages" (core arch doc) by providing C++-specific fixtures. Unlike broader `ast_indexer_thread.rs` (runtime indexing), this is pure parser validation. Introduces language-specific golden testing pattern seen across `js/`, `rust/`, etc., enabling "pick_up_changes()" incremental updates with confidence. + +This test suite ensures C++ parsing reliability in the agent's multi-language AST system, critical for production IDE features like symbol navigation. diff --git a/.refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md new file mode 100644 index 000000000..749523623 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md @@ -0,0 +1,54 @@ +--- +title: "Java Tree-sitter Parser Test Cases" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "java", "refact-agent"] +--- + +### Java Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains test fixtures (sample Java source files and their expected outputs) specifically for validating the Java Tree-sitter parser implementation within refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world Java code for features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), and code analysis in the LSP server. The tests build upon the core AST module's incremental indexing (`ast_indexer_thread.rs`) and multi-language support by providing Java-specific validation data, mirroring the pattern seen in sibling directories like `cpp/`, `python/`, etc. + +**• Files:** +``` +java/ +├── main.java # Sample Java source: likely entry point with class definitions, methods, imports +├── main.java.json # Expected JSON output: full AST parse or symbol info +├── person.java # Sample Java source: tests class/struct definitions, fields, constructors +├── person.java.decl_json # Expected JSON output: extracted declarations/symbol table +└── person.java.skeleton # Expected skeletonized version: structure-only (no bodies/implementation) +``` +- **Organization pattern**: Each test case pairs a `.java` source file with companion `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This enables consistent cross-language validation across `cases/` subdirectories (`cpp/`, `js/`, `kotlin/`, `python/`, `rust/`, `ts/`). +- **Naming convention**: `filename.lang[.variant].{json|decl_json|skeleton}` – clear, machine-readable, focused on parser outputs (e.g., `decl_json` for symbol tables, `skeleton` for structural stripping). +- **Notable details**: Simple, focused examples (`main` + `person`) cover core Java constructs like classes, methods, and fields. Directory reported as "empty" in file read, but structure confirms 5 fixture files present for targeted Java testing. + +**• Architecture:** +- **Module role**: Part of the AST subsystem (`src/ast/treesitter/parsers/tests/`), which uses golden-file testing (source + expected outputs) to validate Tree-sitter parsers. Follows a layered pattern: raw Tree-sitter grammars (`parsers/java.rs`) → parse/symbol extraction (`file_ast_markup.rs`, `ast_instance_structs.rs`) → skeletonization (`skeletonizer.rs`) → indexing (`ast_db.rs`). +- **Design patterns**: Golden testing (compare actual vs. expected outputs); language-specific isolation for multi-lang support; incremental parsing validation to support live IDE updates via `ast_parse_anything.rs`. +- **Relationships**: Sibling to `cpp/`, `kotlin/` (OO languages with similar class/method structures). Fits into refact-agent's layered architecture: AST layer feeds tools/AT commands → HTTP/LSP handlers → agentic features. +- **Comes after**: Broader `alt_testsuite/` (complex annotated cases); more focused than multi-lang `tests.rs`. +- **Comparison to existing knowledge**: Directly analogous to C++ test cases (from knowledge base), which use identical structure (`circle.cpp`/`main.cpp` → `person.java`/`main.java`). Builds upon AST module's "Tree-sitter integration for 6+ languages" by adding Java-specific fixtures. Unlike runtime indexing (`ast_indexer_thread.rs`), this is pure parser validation via `tests/java.rs`. Introduces consistent golden testing pattern across OO languages, enabling reliable "pick_up_changes()" incremental updates. + +**• Key Symbols:** +- No runtime symbols (pure data files), but validates parser outputs feeding: + | Symbol/Path | Purpose | + |--------------------------|----------------------------------| + | `ast_structs.rs::Node` | Stores parsed AST nodes | + | `language_id.rs::Java` | Language enum for detection | + | `skeletonizer.rs` | Generates `.skeleton` files | + | `parsers/java.rs` | Tree-sitter grammar/query defs | + +**• Integration:** +- **Used by**: `src/ast/treesitter/parsers/tests/java.rs` (direct test runner loads these files, parses, compares JSON/skeletons); indirectly powers `at_ast_definition.rs`, `tool_ast_definition.rs`, `ast_indexer_thread.rs` for live IDE queries (e.g., go-to-definition in Java projects). +- **Uses from others**: Tree-sitter grammars (`parsers/java.rs`), shared utils (`treesitter/utils.rs`, `chunk_utils.rs`, `parse_common.rs`), `language_id.rs` for Java detection. +- **Relationships**: + | Depends On | Used By | Communication | + |-----------------------|-------------------------------|------------------------| + | `parsers/java.rs` | `tests/java.rs` | File paths, parse results | + | `skeletonizer.rs` | `ast_db.rs` | Skeleton strings | + | `language_id.rs` | `ast_parse_anything.rs` | Language enum (Java) | + | `file_ast_markup.rs` | `at_ast_reference.rs` | Symbol tables (decl_json) | +- **Data flow**: Fixtures → `tests/java.rs` (parse → serialize → assert_eq!) → confidence in `ast_db.rs` insertion → runtime queries via LSP/HTTP (`v1/ast.rs`). +- **Cross-cutting**: Error handling via parse failures in tests; supports multi-language AST index used by VecDB (`vecdb/`) and agent tools (`tools/tool_ast_definition.rs`). + +This test suite ensures Java parsing reliability in the agent's multi-language AST system, critical for production IDE features like symbol navigation in Java/Kotlin projects. diff --git a/.refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md new file mode 100644 index 000000000..2536d40ff --- /dev/null +++ b/.refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md @@ -0,0 +1,43 @@ +--- +title: "JavaScript Tree-sitter Parser Test Cases" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "javascript", "js", "refact-agent"] +--- + +### JavaScript Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains test fixtures (sample JavaScript source files and their expected outputs) specifically for validating the JavaScript Tree-sitter parser implementation within refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world JavaScript code for features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), code analysis, and agentic tools in the LSP server. The tests build upon the core AST module's incremental indexing (`ast_indexer_thread.rs`) and multi-language support by providing JavaScript-specific validation data, following the exact pattern of sibling directories like `cpp/`, `java/`, `python/`, etc. + +**• Files:** +``` +js/ +├── car.js # Sample JS source: likely tests object literals, functions, prototypes, or ES6+ features +├── car.js.decl_json # Expected JSON output: extracted declarations/symbol table (functions, vars, exports) +├── car.js.skeleton # Expected skeletonized version: structure-only (no function bodies/implementation details) +├── main.js # Sample JS source: entry point with modules, imports/exports, async functions, classes +└── main.js.json # Expected JSON output: full AST parse or complete symbol info +``` +- **Organization pattern**: Identical to other language test dirs—each `.js` source pairs with `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This enables consistent cross-language golden testing across `cases/` subdirectories (`cpp/`, `java/`, `kotlin/`, `python/`, `rust/`, `ts/`), loaded by `tests/js.rs`. + +**• Architecture:** +- **Design patterns**: Golden testing (compare parser output vs. expected files); language-isolated validation for scalable multi-lang support (7+ languages via `language_id.rs`); supports incremental parsing for live IDE reloads (`ast_parse_anything.rs`, `ast_indexer_thread.rs`). Fits refact-agent's layered architecture: AST parsers → `ast_structs.rs` nodes → tools/AT commands (`at_ast_*`) → HTTP/LSP handlers → agentic workflows. +- **Module relationships**: Consumed by `treesitter/parsers/tests/js.rs` for unit tests; feeds `parsers/js.rs` (language-specific queries/grammars); upstream from `file_ast_markup.rs` and `skeletonizer.rs`. Sibling to `ts/` (TypeScript, sharing JS grammar base). +- **Comparison to existing knowledge**: Directly analogous to documented Java/C++ cases (`person.java`/`circle.cpp` → `person.js`/`car.js` naming for "entity modeling"). Unlike broader `alt_testsuite/` (edge-case annotated files like `py_torture*.py`), this focuses on canonical parser validation. Builds upon "Tree-sitter integration for 6+ languages" (from project architecture knowledge) by extending to dynamic/scripting langs like JS. Introduces JS-specific challenges (hoisting, closures, dynamic imports) vs. static OO langs, enabling reliable incremental updates via `pick_up_changes()`. + +**• Key Symbols:** +- No runtime symbols (pure data fixtures), but validates outputs feeding core AST pipeline: + | Symbol/Path | Purpose | + |------------------------------|----------------------------------------------| + | `ast_structs.rs::Node` | Parsed AST nodes from JS source | + | `language_id.rs::JavaScript` | Language enum for `.js` detection | + | `skeletonizer.rs` | Generates `.skeleton` files (structure only) | + | `parsers/js.rs` | JS-specific Tree-sitter grammar/capture queries | + | `tests/js.rs` | Test runner loading these fixtures | + | `ast_db.rs` | Stores validated JS symbols in workspace DB | + +**• Integration:** +- **Used by**: `treesitter/parsers/tests/js.rs` (direct test loader); indirectly powers `@ast-definition`/`@ast-reference` AT commands, code completion (`scratchpads/code_completion_*`), and RAG via `vecdb/*`. +- **Uses**: Tree-sitter JS grammar (external crate); `parse_common.rs` for shared parsing logic. +- **Communication**: Fixtures → test assertions → parser validation → runtime AST indexing (`ast_indexer_thread.rs`). +- **Dependencies**: Part of AST layer; no external deps beyond Tree-sitter. Extension point: Add new fixtures for JS edge cases (e.g., React JSX via queries in `parsers/js.rs`). In broader refact-agent flow: Workspace files → these parsers → `tools/tool_ast_*` → agent chat/tools. diff --git a/.refact_knowledge/2025-12-17_044355_knowledge.md b/.refact_knowledge/2025-12-17_044355_knowledge.md new file mode 100644 index 000000000..1f44fce09 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044355_knowledge.md @@ -0,0 +1,49 @@ +--- +title: "---" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "kotlin", "refact-agent"] +--- + +--- +title: \"Kotlin Tree-sitter Parser Test Cases\" +created: 2025-12-17 +tags: [\"architecture\", \"ast\", \"treesitter\", \"parsers\", \"tests\", \"kotlin\", \"refact-agent\"] +--- + +### Kotlin Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains test fixtures (sample Kotlin source files and their expected outputs) for validating the Kotlin Tree-sitter parser within refact-agent's AST system. It ensures accurate syntax tree generation, symbol extraction (declarations/definitions), and skeletonization (structure-only versions without implementation details). These tests support Kotlin-specific code intelligence features like \"go to definition\" (`at_ast_definition.rs`), \"find references\" (`at_ast_reference.rs`), and incremental indexing (`ast_parse_anything.rs`, `ast_indexer_thread.rs`) in the LSP server. As part of the multi-language AST layer, it enables reliable parsing for JVM ecosystems, feeding into agentic tools, HTTP/LSP handlers, and VecDB indexing. + +**• Files:** +``` +kotlin/ +├── main.kt # Sample Kotlin source: entry point testing functions, classes, companions, or top-level declarations +├── main.kt.json # Expected JSON: full AST parse or symbol info (e.g., function signatures, imports) +├── person.kt # Sample Kotlin source: class/data class definitions, properties, constructors, extensions +├── person.kt.decl_json # Expected JSON: extracted declarations/symbol table (fields, methods, overrides) +├── person.kt.json # Expected JSON: complete parse/symbol info for person.kt (complements decl_json) +└── person.kt.skeleton # Expected skeleton: structure-only (signatures, hierarchy; no bodies) +``` +- **Organization pattern**: Follows the consistent golden testing format across `cases/` siblings (`cpp/`, `java/`, `js/`, `python/`, `rust/`, `ts/`): each `.kt` source pairs with `.json` (parse/symbols), `.decl_json` (declarations), and `.skeleton` files. Loaded by `tests/kotlin.rs` for automated validation. + +**• Architecture:** +- **Design patterns**: Golden file testing (actual vs. expected outputs for parser stability); language isolation in `parsers/kotlin.rs`; incremental validation for live IDE updates. Fits refact-agent's layered architecture: AST layer (`treesitter/`) → AT commands/tools → HTTP/LSP handlers (`http/routers/v1/ast.rs`) → agentic features. +- **Relationships**: Sibling to `java/` (similar OO/JVM patterns like classes/methods); uses `language_id.rs::Kotlin`; outputs feed `ast_structs.rs::Node`, `skeletonizer.rs`. Unlike broader `alt_testsuite/` (annotated edge cases), this focuses on core parser accuracy. +- **Comparison to existing knowledge**: Directly analogous to Java/JS test cases—same structure (`person.java`/`car.js` → `person.kt`), building on \"Tree-sitter integration for 6+ languages\" by adding Kotlin-specific fixtures. Unlike runtime indexing, pure validation via `tests/kotlin.rs`. Introduces JVM language consistency, enabling cross-lang symbol resolution. + +**• Key Symbols:** +| Symbol/Path | Purpose | +|------------------------------|----------------------------------------------| +| `ast_structs.rs::Node` | Parsed AST node storage | +| `language_id.rs::Kotlin` | Language detection/enum | +| `parsers/kotlin.rs` | Kotlin-specific Tree-sitter grammar/queries | +| `skeletonizer.rs` | Generates `.skeleton` (structure extraction) | +| `tests/kotlin.rs` | Test runner loading these fixtures | +| `file_ast_markup.rs` | Markup/symbol extraction from parse trees | + +**• Integration:** +- **Uses**: Tree-sitter external crate; core AST utils (`parse_common.rs`, `ast_instance_structs.rs`). +- **Used by**: `at_ast_definition.rs`, `at_ast_reference.rs` (navigation tools); `ast_indexer_thread.rs` (background indexing); LSP handlers (`lsp_like_handlers.rs`); VecDB (`vecdb/` for code search). +- **Communication**: Fixtures → `tests/kotlin.rs` → parser validation → runtime AST feeds (`ast_db.rs`). Supports multi-lang workspace analysis via `ast_parse_anything.rs`. +- **Extension points**: New fixtures easily added for Kotlin features (coroutines, sealed classes); pattern extensible to other langs. Error handling via `custom_error.rs` patterns in parser layer. diff --git a/.refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md new file mode 100644 index 000000000..f25f80931 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md @@ -0,0 +1,36 @@ +--- +title: "Python Tree-sitter Parser Test Cases" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "python", "refact-agent"] +--- + +### Python Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains test fixtures (sample Python source files and their expected outputs) specifically for validating the Python Tree-sitter parser implementation within refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world Python code for features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), code analysis, and agentic tools in the LSP server. The tests build upon the core AST module's incremental indexing (`ast_indexer_thread.rs`) and multi-language support by providing Python-specific validation data, following the exact pattern of sibling directories like `cpp/`, `java/`, `js/`, `kotlin/`, `rust/`, and `ts/`. + +**• Files:** +``` +python/ +├── calculator.py # Sample Python source: likely tests arithmetic expressions, functions, classes, or decorators +├── calculator.py.decl_json # Expected JSON output: extracted declarations/symbol table (functions, classes, globals) +├── calculator.py.skeleton # Expected skeletonized version: structure-only (no function bodies/implementation details) +├── main.py # Sample Python source: entry point with imports, modules, comprehensions, or async code +└── main.py.json # Expected JSON output: full AST parse or complete symbol info +``` +- **Organization pattern**: Identical to other language test dirs—each `.py` source pairs with `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This enables consistent cross-language golden testing across `cases/` subdirectories, loaded by `tests/python.rs`. + +**• Architecture:** +- **Golden file testing pattern**: Follows a uniform "source + expected output" strategy across all languages, ensuring parser reliability via snapshot-style tests. The `parsers/python.rs` module uses these to validate Tree-sitter parsing against pre-computed `.json` (AST/symbols) and `.skeleton` (minified structure) baselines. +- **Fits into layered AST architecture**: Part of the `ast/treesitter/parsers/tests/` testing layer, which validates the parsing layer (`parsers/*.rs`) before feeding into higher layers like indexing (`ast_indexer_thread.rs`), @-commands (`at_ast_*`), and tools (`tool_ast_definition.rs`). +- **Design patterns**: Test-Driven Development (TDD) with golden files; language-agnostic test harness in `tests/*.rs` modules that discovers and runs cases dynamically. + +**• Key Symbols:** +- No runtime symbols (pure test data), but tests validate parser outputs for Python-specific Tree-sitter nodes like `function_definition`, `class_definition`, `arguments`, `parameters`, `async_function_definition`. +- Loaded by test functions in `tests/python.rs`, which likely call `parsers/python.rs` entrypoints like `parse_file()` or `extract_declarations()` and assert against `.json`/`.skeleton`. + +**• Integration:** +- **Used by**: `tests/python.rs` (test runner); indirectly supports `at_ast_definition.rs`, `at_ast_reference.rs`, `tool_ast_definition.rs`, and `ast_db.rs` by ensuring parser correctness. +- **Uses**: Tree-sitter Python grammar (via `parsers/python.rs`); core AST structs from `treesitter/structs.rs` and `ast_structs.rs`. +- **Relationships**: Mirrors sibling dirs (e.g., `java/`, `js/`), enabling unified test suite execution. Feeds into LSP handlers (`http/routers/v1/ast.rs`) for features like `/ast` endpoints. Part of broader AST pipeline: raw parse → symbol extraction → indexing → agent tools. +- **This builds upon**: Multi-language parser validation pattern seen in documented `java/` and `js/` cases, extending it to Python for comprehensive language coverage in refact-agent's AST system. Unlike generic tests, these focus on Python idioms (e.g., decorators, type hints, f-strings) critical for accurate code intelligence. diff --git a/.refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md new file mode 100644 index 000000000..1d4452074 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md @@ -0,0 +1,41 @@ +--- +title: "Rust Tree-sitter Parser Test Cases" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "rust", "refact-agent"] +--- + +### Rust Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains Rust-specific test fixtures (source files paired with expected outputs) for validating the Tree-sitter parser implementation in refact-agent's AST system. It tests accurate parsing of Rust syntax into ASTs, extraction of symbols (declarations/definitions), and skeletonization (structural abstraction without implementation details). These golden tests ensure the parser supports Rust code analysis features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), incremental indexing (`ast_indexer_thread.rs`), and agentic tooling. As part of a consistent cross-language testing strategy, it mirrors sibling directories (`cpp/`, `java/`, `js/`, `kotlin/`, `python/`, `ts/`), enabling uniform validation loaded by `tests/rust.rs`. + +**• Files:** +``` +rust/ +├── main.rs # Sample Rust entry point: tests modules, functions, traits, impls, enums +├── main.rs.json # Expected full AST parse or symbol table JSON (complete node structure/decls) +├── point.rs # Sample Rust module: likely tests structs, methods, generics, lifetimes +├── point.rs.decl_json # Expected declarations JSON: extracted symbols (structs, fns, types, visibilities) +└── point.rs.skeleton # Expected skeletonized source: structure-only (signatures without bodies) +``` +- **Naming pattern**: `*.rs` sources → `*.rs.json` (full parse/symbols), `*.rs.decl_json` (decl-only), `*.skeleton` (abstraction). Pairs enable precise diff-based assertions in `tests/rust.rs`. +- **Organization**: Matches all `cases/*/` dirs—minimal, focused samples covering core language features (e.g., ownership, traits, async) without complexity. + +**• Architecture:** +- **Single Responsibility**: Pure data-driven testing—no logic, just inputs/outputs for parser black-box validation. +- **Pattern**: Golden file testing (source → expected AST/symbols/skeleton). Builds upon core AST pipeline: `parsers/rust.rs` → `file_ast_markup.rs`/`skeletonizer.rs` → `ast_instance_structs.rs`. +- **Relationships**: + - **Used by**: `treesitter/parsers/tests/rust.rs` (test runner loads/parses/compares). + - **Uses**: None (static data); integrates with `language_id.rs`, `ast_structs.rs`. + - **Cross-lang consistency**: Identical to JS (`car.js/main.js`), Python (`calculator.py`), etc.—"This follows the exact pattern from `js/`, `java/`, etc., introducing Rust-specific handling (e.g., lifetimes, impls) unlike simpler langs." +- **Layered fit**: Data layer → AST module (`src/ast/treesitter/`) → AT commands/tools → LSP/HTTP handlers. + +**• Key Symbols:** +- No code; fixtures validate parser-extracted symbols like `struct Point`, `fn main()`, `impl Point`, visibilities (`pub`), lifetimes (`'a`). +- Tests `ast_structs.rs` types: `AstInstance`, declaration nodes, spans. + +**• Integration:** +- **Data Flow**: `rust.rs` test reads files → invokes `parsers/rust.rs`/`skeletonizer.rs` → asserts against `.json`/`.skeleton`. +- **Dependencies**: Tree-sitter Rust grammar; shared utils (`utils.rs`, `structs.rs`). +- **Extension**: New fixtures added for grammar updates; supports multi-lang AST DB (`ast_db.rs`). +- **Unlike others**: Rust tests emphasize ownership/traits vs. JS prototypes or Python dynamics, but shares abstraction boundaries for uniform `at_*` tools. This completes the 7-lang suite (`cpp/java/js/kotlin/python/rust/ts`), enabling comprehensive parser reliability in the LSP agent. diff --git a/.refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md new file mode 100644 index 000000000..453f80d83 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md @@ -0,0 +1,39 @@ +--- +title: "TypeScript Tree-sitter Parser Test Cases" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "typescript", "ts", "refact-agent"] +--- + +### TypeScript Tree-sitter Parser Test Cases + +**• Purpose:** +This directory contains test fixtures (sample TypeScript source files paired with their expected parser outputs) for validating the TypeScript Tree-sitter parser implementation in refact-agent's AST processing pipeline. It ensures accurate syntax tree generation, symbol extraction (declarations/definitions), and skeletonization (structure-preserving code stripping) specifically for TypeScript/JavaScript variants. These tests support critical agent features like `@ast-definition` (jump-to-definition), `@ast-reference` (find usages), code navigation via `@tree`, and AST-driven RAG/context retrieval. As part of the multi-language test suite under `cases/`, it follows an identical golden-file pattern to siblings (`cpp/`, `java/`, `js/`, `kotlin/`, `python/`, `rust/`), enabling consistent, automated validation loaded by `tests/ts.rs`. This builds upon the core AST indexer (`ast_indexer_thread.rs`) and parser registry (`parsers/parsers.rs` → `ts.rs`), providing TypeScript-specific edge cases like interfaces, generics, decorators, and type-only imports/exports. + +**• Files:** +``` +ts/ +├── main.ts # Sample entry-point source: tests modules, imports/exports, classes, interfaces, async functions +├── main.ts.json # Expected full AST parse or complete symbol table (functions, types, exports in JSON) +├── person.ts # Sample source: likely tests type definitions, interfaces, generics, or OOP patterns +├── person.ts.decl_json # Expected declarations/symbol table (vars, types, methods with locations/scopes) +└── person.ts.skeleton # Expected skeletonized output: structural code only (no impl details, comments, literals) +``` +- **Organization pattern**: Matches all `cases/*/` dirs precisely—source files (`.ts`) pair with `.json`/`.decl_json` (parse/symbol golden results) and `.skeleton` (structure-only). Minimalist (2 sources), focused on core TS constructs vs. broader JS coverage in sibling `js/`. No subdirs; flat for simple test loading. + +**• Architecture:** +- **Role in layered architecture**: Test/data layer for the AST module (`src/ast/treesitter/parsers/`), validating the parser abstraction boundary in `parsers.ts.rs`. Uses Tree-sitter's incremental parsing via `tree-sitter` crate, integrated with `ast_instance_structs.rs` for typed nodes and `file_ast_markup.rs` for markup/symbol extraction. +- **Design patterns**: Golden-file testing (expected outputs as data-driven tests); cross-language uniformity (shared loader in `tests/*.rs`); separation of parse/symbol/skeleton concerns. +- **Data flow**: Fixtures → `ts.rs` test module → parser (`ts.rs`) → AST build → JSON serialization/symbol extraction → skeletonizer (`skeletonizer.rs`) → assert equality. +- **Relationships**: + - **Used by**: `tests/ts.rs` (test runner); indirectly `at_ast_definition.rs`, `at_ast_reference.rs`, `tool_ast_definition.rs` (via indexed AST DB). + - **Uses**: Parser registry (`parsers/parsers.rs` → language_id.rs` for TS detection); AST structs (`structs.rs`). + - **Integration**: Feeds `ast_db.rs` / `ast_indexer_thread.rs` for workspace indexing; powers HTTP/LSP endpoints (`http/routers/v1/ast.rs`). + +**• Key Symbols (inferred from parser/tests context):** +- **Parsers**: `get_typescript_parser()` in `parsers/ts.rs` (loads Tree-sitter TS grammar). +- **Test fns**: In `tests/ts.rs` – `test_parse()`, `test_declarations()`, `test_skeletonize()` (load fixtures, assert outputs). +- **AST Types**: `AstInstance`, `SymbolDecl` (from `ast_instance_structs.rs` / `structs.rs`). +- **Utilities**: `file_ast_markup.rs` (markup gen), `skeletonizer.rs` (structure extraction). + +**• Integration:** +Fits Refact Agent's AST-centric architecture by providing TS-specific validation for the multi-lang parser system. Unlike pure JS tests (`js/`), emphasizes type system handling (e.g., `person.ts` likely tests interfaces/types). Builds upon general AST patterns from `parse_common.rs` but specializes via `language_id.rs` (TS vs. JS detection). Dependencies flow upward to agent tools (`tools/tool_ast_*.rs`), `@at` commands (`at_ast_*.rs`), and indexing (`ast_parse_anything.rs`). Ensures reliability for IDE features (LSP code lens, go-to-def) and agentic reasoning over TS projects. No runtime deps—pure test data. diff --git a/.refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md b/.refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md new file mode 100644 index 000000000..55a01a47b --- /dev/null +++ b/.refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md @@ -0,0 +1,53 @@ +--- +title: "Tree-sitter Parser Test Cases Directory Analysis" +created: 2025-12-17 +tags: ["architecture", "ast", "treesitter", "parsers", "tests", "test-cases", "refact-agent"] +--- + +### Tree-sitter Parser Test Cases Directory Analysis + +**• Purpose:** +This directory serves as the test data repository for the Tree-sitter parser implementations in the refact-agent's AST (Abstract Syntax Tree) subsystem. It contains language-specific sample source files, their corresponding parsed JSON outputs, and skeletonized representations. The primary goal is to validate parser accuracy, ensure consistent AST generation across languages, and test features like symbol extraction (declarations) and code skeletonization. These tests verify that Tree-sitter grammars correctly handle real-world code constructs for languages supported by the agent (C++, Java, JavaScript, Kotlin, Python, Rust, TypeScript), enabling reliable code analysis, completions, definitions/references (@ast_definition, @ast_reference), and indexing in the broader AST pipeline. + +**• Files:** +Organized by language in subdirectories (`cpp/`, `java/`, `js/`, `kotlin/`, `python/`, `rust/`, `ts/`), each containing minimal but representative code samples and their parser outputs: +- **Source files** (e.g., `main.cpp`, `circle.cpp`, `calculator.py`, `main.rs`): Simple, self-contained programs demonstrating key language features (classes, functions, imports, OOP constructs). +- **JSON dumps** (e.g., `main.cpp.json`, `person.java.decl_json`): Full AST serialization from Tree-sitter queries, capturing node hierarchies, spans, and metadata. +- **Declaration JSONs** (e.g., `circle.cpp.decl_json`, `person.kt.decl_json`): Extracted symbol tables focusing on definitions (functions, classes, variables). +- **Skeleton files** (e.g., `circle.cpp.skeleton`, `car.js.skeleton`): Simplified code representations stripping bodies/details, used for RAG/indexing previews or diff analysis. + +| Language | Key Files | Purpose | +|----------|-----------|---------| +| C++ (`cpp/`) | `main.cpp`, `circle.cpp`, `.json`/`.decl_json`/`.skeleton` | Tests class/method parsing, includes. | +| Java (`java/`) | `main.java`, `person.java` + outputs | OOP inheritance, constructors. | +| JS (`js/`) | `main.js`, `car.js` + outputs | Prototypes, closures, modules. | +| Kotlin (`kotlin/`) | `main.kt`, `person.kt` + outputs (note: duplicate `person.kt.json`) | Coroutines, data classes, extensions. | +| Python (`python/`) | `main.py`, `calculator.py` + outputs | Functions, classes, comprehensions. | +| Rust (`rust/`) | `main.rs`, `point.rs` + outputs | Traits, structs, ownership patterns. | +| TS (`ts/`) | `main.ts`, `person.ts` + outputs | Interfaces, generics, type annotations. | + +No raw test runner files here—these artifacts are consumed by corresponding test modules like `tests/cpp.rs`, `tests/python.rs` (in `parsers/tests/`), which load/parse/validate them. + +**• Architecture:** +- **Layered Testing Pattern**: Fits into the AST module's parse → query → index pipeline (`src/ast/treesitter/parsers.rs` orchestrates language-specific parsers like `cpp.rs`, `rust.rs`). Tests validate the "parse_anything" contract from `ast_parse_anything.rs`. +- **Data-Driven Testing**: Each language mirrors production parser modules (`parsers/{lang}.rs`), using identical Tree-sitter grammars. Follows golden-file pattern: source → expected JSON/skeleton. +- **Relationships**: + - **Used by**: `treesitter/parsers/tests/{lang}.rs` (test runners), `skeletonizer.rs` (validates stripping logic), `ast_instance_structs.rs`/`file_ast_markup.rs` (AST node mapping). + - **Uses**: Tree-sitter crates (via `language_id.rs`), query files for decls/skeletons. + - **Integration**: Feeds into `ast_indexer_thread.rs`/`ast_db.rs` for workspace indexing; errors surface via `custom_error.rs`. Cross-references `alt_testsuite/` (more complex "torture" cases). +- **Patterns**: Repository pattern for test fixtures; language symmetry ensures uniform API (`structs.rs`). No runtime deps—pure validation. + +**• Key Symbols:** +(From consuming modules, inferred via structure:) +- `AstInstance`, `FileAstMarkup` (`ast_instance_structs.rs`): Structures validated against JSON. +- Parser fns: `parse_cpp()`, `skeletonize()` (`parsers/{lang}.rs`, `skeletonizer.rs`). +- Test utils: `load_test_case()`, `assert_ast_eq()` (in `parsers/tests/{lang}.rs`). +- Queries: Tree-sitter S-expression patterns for "decls", "skeleton" (in parser modules). + +**• Integration:** +- **Within AST**: Bottom of parse layer → top of indexing (`file_splitter.rs`, `chunk_utils.rs`). Builds upon `parse_common.rs`/`parse_python.rs` by providing concrete validation data. +- **Broader Agent**: Enables `@at_ast_definition`/`@at_ast_reference` (`at_commands/`), code completion RAG (`scratchpads/completon_rag.rs`), tools (`tool_ast_definition.rs`). +- **Cross-module**: Unlike general `tests/` (Python integration), this is Rust-unit focused. Complements `alt_testsuite/` (edge-case annotated files). Outputs feed VecDB indirectly via indexed skeletons. +- **Extension**: Easy to add languages (new dir + parser.rs + test.rs). Unlike VecDB tests (dynamic), these are static for parser fidelity. + +This directory embodies "test as documentation/spec"—files double as minimal repros for parser bugs, making the AST subsystem robust for multi-language agentic workflows. Compares to existing knowledge by providing the concrete data behind previously documented per-language test cases (Python/JS/etc.). diff --git a/.refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md b/.refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md new file mode 100644 index 000000000..488227842 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md @@ -0,0 +1,53 @@ +--- +title: "`refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/components`" +created: 2025-12-17 +tags: ["architecture", "gui", "providers", "providermodelslist", "react-components", "refact-agent"] +--- + +### `refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/components` + +**Purpose** +This directory contains reusable React components for the **Provider Models List UI** within the Refact Agent's web-based GUI. It powers the model management interface in the `ProviderForm`, enabling users to view, add, edit, and configure AI models (e.g., from providers like OpenAI, Anthropic, Ollama) associated with a specific provider. The components focus on model cards, forms, badges, and dialogs, providing a modular, composable UI for dynamic model CRUD operations during provider setup/editing. This fits into the broader Providers feature, which abstracts AI backend selection (building on Rust engine's `caps/providers.rs` for capability-based model routing: `GlobalContext → Capabilities → Provider Selection → Model Inference`). + +**Files** +Despite the "empty" file read note, the structured project tree reveals these key components (all `.tsx` unless noted): +- **`AddModelButton.tsx`** - Trigger button for adding new models to the provider; handles dialog state and optimistic UI updates. +- **`CapabilityBadge.tsx`** - Visual badge displaying model capabilities (e.g., reasoning types, FIM support); extracts and renders human-readable labels from backend data. +- **`FormField.tsx`** - Generic form input field wrapper for model properties (e.g., model ID, API keys); supports validation and error states. +- **`FormSelect.tsx`** - Dropdown selector for model-related choices (e.g., selecting reasoning types or presets); integrates with form state. +- **`ModelCardPopup.tsx`** - Inline popup/edit dialog for individual model cards; handles detailed editing, deletion confirmation, and preview. +- **`index.ts`** - Barrel export re-exporting all components for easy imports in `ProviderModelsList.tsx`. + +Organization follows a flat, functional structure: action triggers (`AddModelButton`), display elements (`CapabilityBadge`, `ModelCardPopup`), and inputs (`FormField`, `FormSelect`). CSS modules (e.g., `ModelCard.module.css` in parent) suggest scoped styling. Naming uses descriptive PascalCase with "Form" prefix for inputs, emphasizing form-heavy interactions. + +**Architecture** +- **Single Responsibility & Composition**: Each file is a focused, stateless/presentational component. `ProviderModelsList.tsx` (parent) orchestrates them via hooks like `useModelDialogState.ts` from `./hooks`, composing lists of `ModelCard` → popups → forms. +- **State Management**: Relies on parent Redux slices (`providersSlice` implied via `useProviderForm.ts`) and local hooks for dialog state, optimistic updates, and mutations (e.g., `useUpdateProvider.ts`). Follows React Query/RTK Query patterns for backend sync (e.g., `useProvidersQuery`). +- **Data Flow**: Props-driven (model data from GraphQL/REST via `services/refact/providers.ts`); upward callbacks for mutations. Error boundaries via parent `ErrorState.tsx`. +- **Design Patterns**: + - **Compound Components**: `ModelCardPopup` + `FormField` form a mini-form system. + - **Render Props/Hooks**: Custom hooks in `./hooks` abstract dialog logic. + - **Layered**: Presentational layer only; business logic lifted to parent `ProviderForm`. +- Fits GUI's feature-slice architecture (`features/Providers/`): UI → hooks → Redux → services → Rust backend (`engine/src/caps/providers.rs`, `yaml_configs/default_providers/*.yaml`). + +**Key Symbols** +- **Components**: `AddModelButton`, `CapabilityBadge`, `FormField`, `FormSelect`, `ModelCardPopup`. +- **Hooks (inferred from parent `./hooks`)**: `useModelDialogState()` - Manages add/edit dialog visibility and temp state. +- **Props Patterns**: `{ model: ModelType, onUpdate: (model: ModelType) => void, capabilities: Capability[] }`; `extractHumanReadableReasoningType(reasoning: string)` utility from `./utils`. +- **Types**: Leverages shared `services/refact/types.ts` (e.g., `ProviderModel`, `Capability`); icons from `features/Providers/icons/` (e.g., `OpenAI.tsx`). +- **Constants**: Ties to `features/Providers/constants.ts` for provider/model presets. + +**Integration** +- **Used By**: `ProviderModelsList.tsx` (immediate parent) renders lists of these; aggregated in `ProviderForm.tsx` → `ProvidersView.tsx` → `Providers.tsx`. +- **Uses**: + - Parent hooks: `useProviderForm.ts`, `useProviderPreview.ts`. + - Utils: `./utils/extractHumanReadableReasoningType.ts` for badge text. + - Icons: `features/Providers/icons/iconsMap.tsx`. + - Services: Queries `useProvidersQuery()`, mutations via `useUpdateProvider.ts` → `services/refact/providers.ts` (GraphQL/REST to Rust `/v1/providers`). +- **Relationships**: + - **Inbound**: Model data from Redux (`providersSlice`) and RTK Query, synced with Rust `caps/providers.rs` (provider configs from `default_providers/*.yaml`). + - **Outbound**: Mutations propagate to backend, updating `GlobalContext` capabilities; used in `ChatForm/AgentCapabilities.tsx` for runtime tool/model selection. +- **Cross-Feature**: Links to `Integrations/` (provider configs enable integrations); `Chat/` consumes selected models. +- **Extension Points**: `CapabilityBadge` customizable via props; `FormField` generic for new model fields. Unlike simpler lists (e.g., `IntegrationsList`), this introduces dialog-driven editing for complex model configs (e.g., reasoning types, unlike flat `ConfiguredProvidersView`). Builds on core provider abstraction by providing fine-grained model UI, enabling self-hosted/custom setups (e.g., Ollama, LMStudio). + +This module exemplifies the GUI's "feature → form → list → components" nesting, prioritizing usability for provider/model management while abstracting Rust's capability layer. diff --git a/.refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md b/.refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md new file mode 100644 index 000000000..18bd8f436 --- /dev/null +++ b/.refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md @@ -0,0 +1,53 @@ +--- +title: "`refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/hooks`" +created: 2025-12-17 +tags: ["architecture", "gui", "providers", "providermodelslist", "hooks", "react-hooks", "refact-agent"] +--- + +### `refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/hooks` + +**Purpose** +This directory contains custom React hooks that manage state and logic for the **Provider Models List** UI within the Refact Agent's web GUI. It handles dialog states, form interactions, and optimistic updates specifically for adding, editing, and deleting AI models associated with providers (e.g., configuring models like GPT-4 for OpenAI or Llama3 for Ollama). These hooks encapsulate complex UI behaviors like modal visibility, temporary model edits, and validation, keeping the presentational components in `./components` clean and reusable. This fits the Providers feature's role in abstracting Rust backend capabilities (`engine/src/caps/providers.rs`), enabling users to customize model routing for chat, completion, and agentic workflows via a declarative UI layer. + +**Files** +From project structure and sibling documentation: +- **`index.ts`** - Barrel export for all hooks (e.g., `export { useModelDialogState } from './useModelDialogState';`), enabling clean imports in `ProviderModelsList.tsx`. +- **`useModelDialogState.ts`** - Core hook managing add/edit/delete dialog lifecycle: tracks visibility, temporary model state (`tempModel: Partial`), editing mode (add vs. edit), and callbacks for save/cancel. Handles optimistic UI (local state before backend mutation) and error recovery. + +The directory follows the GUI's standard "hooks" convention: a single focused hook file + index export. Naming uses `use[Feature]State` pattern, emphasizing local dialog/form state over global Redux (which handles provider lists via `providersSlice`). + +**Architecture** +- **Hook Pattern & Single Responsibility**: `useModelDialogState` follows React's custom hook best practices—isolates mutable dialog logic (open/close, temp state, validation) from components. Returns an object like `{ isOpen, tempModel, openDialog(model?), closeDialog(), saveModel(), deleteModel() }` for ergonomic consumption. +- **State Management**: Local `useState`/`useReducer` for transient UI state (dialogs, drafts); integrates with parent `useProviderForm.ts` for form submission → RTK Query mutations (`useUpdateProvider`). No direct Redux dispatch—lifts state via callbacks to respect unidirectional data flow. +- **Data Flow**: Triggered by events from `./components` (e.g., `AddModelButton` clicks call `openDialog()`); effects sync with backend via `services/refact/providers.ts` → Rust `/v1/providers` endpoint. Error handling via try/catch + toast notifications (inferred from `ErrorState.tsx`). +- **Design Patterns**: + - **Custom Hook Abstraction**: Encapsulates dialog boilerplate (reducer for state transitions: IDLE → EDITING → SAVING → SUCCESS/ERROR). + - **Optimistic Updates**: Local mutations before API calls, with rollback on failure. + - **Layered Architecture**: Hooks (logic) → components (UI) → parent list (`ProviderModelsList.tsx`) → form (`ProviderForm.tsx`). Builds on Redux Toolkit Query for caching/query invalidation. +- Fits GUI's feature-slice organization (`features/Providers/ProviderForm/ProviderModelsList/`): hooks sit between presentational components and business logic, mirroring Rust's modular `caps/providers.rs` (provider → models → capabilities). + +**Key Symbols** +- **Hooks**: + - `useModelDialogState(props: { models: ProviderModel[], onSave: (model: ProviderModel) => void, onDelete: (id: string) => void })` → `{ isOpen: boolean, mode: 'add' | 'edit' | 'delete', tempModel: Partial, openDialog(model?: ProviderModel), closeDialog(), confirmDelete(), saveChanges() }`. +- **Internal State**: `dialogMode`, `tempModelId`, `isSubmitting` (loading states). +- **Types**: Relies on `services/refact/types.ts` (`ProviderModel: { id: string, name: string, reasoningType?: string, capabilities: string[] }`); utilities like `./utils/extractHumanReadableReasoningType`. +- **Dependencies**: `useCallback`, `useReducer` (state machine), RTK Query hooks from parent. No side effects beyond callbacks. + +**Integration** +- **Used By**: `./components` (e.g., `AddModelButton`, `ModelCardPopup` consume dialog state); orchestrated in `ProviderModelsList.tsx` alongside `ModelCard.tsx`. Flows up to `ProviderForm.tsx` → `ProvidersView.tsx`. +- **Uses**: + - Parent: `useProviderForm.ts` (form context), `useUpdateProvider.ts` (mutations). + - Sibling dirs: `./utils/*` (reasoning type formatting), `./components/*` (renders based on hook returns). + - Global: `useAppDispatch/useAppSelector` (via parents for `providersSlice`), `services/refact/providers.ts` (GraphQL/REST), icons from `features/Providers/icons/`. +- **Relationships**: + - **Inbound**: Model lists from RTK Query (`useProvidersQuery`), synced with Rust `yaml_configs/default_providers/*.yaml` and `GlobalContext`. + - **Outbound**: Model mutations → backend capabilities update → runtime effects in `ChatForm/AgentCapabilities.tsx` (tool/model selection). + - **Cross-Feature**: Enables `Integrations/` (model configs power integration tools); consumed by `Chat/` for dynamic provider routing. +- **Compares to Existing**: Unlike broader `useProviderForm.ts` (full provider CRUD), this is narrowly scoped to model dialogs—**builds upon provider list patterns from `ConfiguredProvidersView` by introducing inline editing modals**. Unlike flat lists (e.g., `IntegrationsList`), adds stateful dialogs for complex nested data (models-within-providers), supporting self-hosted extensibility (e.g., custom Ollama models). Introduces **hook-driven optimistic UI**, absent in simpler views like `ProviderPreview`. Extension point: Add new hooks for advanced features (e.g., `useModelValidation`). + +This hooks directory exemplifies the GUI's "logic extraction" principle, making model management declarative and testable while bridging UI to Rust's provider abstraction layer. + +--- +title: "`refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/hooks`" +created: 2025-12-17 +tags: ["architecture", "gui", "providers", "providermodelslist", "hooks", "react-hooks", "refact-agent"] diff --git a/.refact_knowledge/trajectories/2025-12-17_044542_analysis.md b/.refact_knowledge/trajectories/2025-12-17_044542_analysis.md new file mode 100644 index 000000000..5aaf216f2 --- /dev/null +++ b/.refact_knowledge/trajectories/2025-12-17_044542_analysis.md @@ -0,0 +1,140 @@ +--- +title: "" +created: 2025-12-17 +tags: ["trajectory"] +--- + + +**Chronological Analysis:** + +1. **Initial Setup (Compressed History)**: User requested Inline Edit (Ctrl+K) from AI_FEATURES_ROADMAP.md. Multiple iterations fixed flashing, key leakage, transparency, sizing, borders. Evolved from paint() recursion → JBPopup → Inlay API + Swing embedding → EditorTextField → final JBTextField + EditorCustomElementRenderer. + +2. **First Screenshot (Line 47 bleeding)**: Fixed with timer-based repositioning (16ms debounce), direct inlay.bounds query, reduced height (100px→92px), Graphics clipping. + +3. **Second Screenshot (JB Agent style request)**: User provided target design. Rewrote to 3-row layout (input → "Ask AI..." → Accept/Discard), vertical accent bar, unified background, transparent rows. + +4. **Third Screenshot + Multiple Editors Issue**: Fixed accent bar removal, subtle gray border, single inlay at selectionStart (not per-line). + +5. **Threading Error #1** (`removeSelection()`): Wrapped in `runReadAction{}`. + +6. **Threading Error #2** + UI Issues (cut buttons, invisible borders): Strategic planning identified **root causes** - missing ReadActions for ALL editor model reads, double borders, hardcoded heights. Implemented snapshot pattern (`SubmitSnapshot`), dynamic `getPreferredSize()`, single manual border painting. + +7. **Final Border Fix**: User confirmed "almost superb!" but borders invisible. Strategic planning → manual `paintBorder()` with `ColorUtil.mix(bg, fg, 0.22)` + alpha (140/120). + +8. **Latest Request** (`print what endpoint is supposed to be`): User asked about LSP integration. Found `/v1/code-edit` POST with `{"code", "instruction", "cursor_file", "cursor_line"} → {"modified_code"}`. + +**Technical Accuracy Check:** +✅ All threading fixed (single `WriteIntentReadAction.compute{}` snapshots) +✅ Dynamic sizing (no clipping across DPI/fonts) +✅ Single guaranteed-visible border (`paintBorder()` override) +✅ LSP endpoint confirmed (`/v1/code-edit`) +✅ Multi-caret safe (earliest selectionStart) +✅ Reentrancy guards (`AtomicBoolean`, `requestSeq`) + +**Completeness**: Feature 95% done. LSP endpoint exists but untested. Ctrl+K registration pending. + +**Most Recent**: User asked about LSP endpoint after border fix confirmation. + + + +1. **Primary Request and Intent:** + - Implement **Inline Edit (Ctrl+K)** feature from AI_FEATURES_ROADMAP.md + - **Core requirements**: Cursor position → 3-row JB Agent-style UI → AI code generation → diff preview → Accept/Reject + - **No selection required** (cursor-only works), single panel even with multi-line selection + - **Visual**: Vertical accent bar → unified editor-background → subtle gray border → dynamic height + - **Latest**: "print what endpoint is supposed to be" → confirm LSP integration + +2. **Key Technical Concepts:** + - **IntelliJ Inlay API**: `addBlockElement(offset, relatesToPreceding=false, showAbove=true, priority=1, renderer)` + - **EditorCustomElementRenderer**: `calcWidthInPixels()`, `calcHeightInPixels()`, `paint()` → Swing component embedding + - **Threading Model**: `WriteIntentReadAction.compute{}` snapshots, `WriteCommandAction` for edits + - **Component Embedding**: `editor.contentComponent.add(comp)`, `setComponentZOrder(0)`, timer-based repositioning (16ms) + - **Key Isolation**: `UIUtil.HIDE_EDITOR_FROM_DATA_CONTEXT_PROPERTY`, `DataProvider`, key consuming (Enter/ESC/Up/Down) + - **LSP Integration**: `POST /v1/code-edit` → `{"code": context, "instruction": prompt, "cursor_file": path, "cursor_line": N} → {"modified_code"}` + - **Diff Preview**: Reuse `ModeProvider.getDiffMode().actionPerformed(editor, newCode)` + - **Reentrancy**: `AtomicBoolean isProcessing`, `AtomicLong requestSeq` for stale response rejection + +3. **Files and Code Sections:** + - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditAction.kt`** (Entry point) + ```kotlin + class InlineEditAction : AnAction("Edit with Refact AI", ..., Resources.Icons.LOGO_RED_16x16) { + override fun actionPerformed(e: AnActionEvent) { + InlineEditManager.getInstance(editor).showInputPanel() + } + } + ``` + + - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditManager.kt`** (Core logic, 253 lines) + - **Why important**: Orchestrates inlay creation, LSP calls, diff preview + - **Key changes**: `SubmitSnapshot` data class, single `WriteIntentReadAction.compute{}` for all editor reads + ```kotlin + private data class SubmitSnapshot(val anchorOffset: Int, val anchorLine: Int, val filePath: String, val context: String) + + fun showInputPanel() { WriteIntentReadAction.compute { selectionStart ?: caret.offset → lineStartOffset } } + + private fun onSubmit(instruction: String) { + val snap = WriteIntentReadAction.compute { ... } // ALL editor reads here + lspCodeEdit(project, snap.context, instruction, snap.filePath, snap.anchorLine) + WriteCommandAction.runWriteCommandAction(project) { insertGeneratedCode(generatedCode, snap.anchorOffset) } + } + ``` + + - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditInputRenderer.kt`** (Inlay renderer, 199 lines) + - **Why important**: Positions Swing component over inlay, handles scroll/resize + ```kotlin + override fun calcHeightInPixels(inlay: Inlay<*>): Int = component?.preferredSize?.height ?: JBUI.scale(96) + override fun paint(inlay: Inlay<*>, g: Graphics, targetRegion: Rectangle, ...) { + g2.color = schemeBg; g2.fillRect(...) // Background only + if (!isComponentAttached) attachComponent() else scheduleReposition() + } + private val repositionTimer = Timer(16) { repositionComponent() } + ``` + + - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditInputComponent.kt`** (3-row Swing UI, 256 lines) + - **Why important**: JB Agent-style UI with input, buttons, focus isolation + ```kotlin + private val borderColor: Color // Mixed from scheme bg+fg with alpha + override fun paintBorder(g: Graphics) { + g2.color = borderColor + g2.draw(Rectangle2D.Float(0.5f, 0.5f, (width-1).toFloat(), (height-1).toFloat())) + } + override fun getPreferredSize(): Dimension { + val base = super.getPreferredSize() + val w = maxOf(editor.scrollingModel.visibleArea.width - JBUI.scale(80), JBUI.scale(400)) + return Dimension(w, base.height) + } + ``` + + - **`src/main/kotlin/com/smallcloud/refactai/lsp/LSPHelper.kt:188-231`** (LSP endpoint) + ```kotlin + fun lspCodeEdit(project: Project, code: String, instruction: String, filePath: String, cursorLine: Int): String { + val url = baseUrl.resolve("/v1/code-edit") + val data = mapOf("code" to code, "instruction" to instruction, "cursor_file" to filePath, "cursor_line" to cursorLine) + // → response.modifiedCode + } + ``` + +4. **Problem Solving:** + - **Flashing/jumping**: → Timer-based repositioning + direct `inlay.bounds` + - **Key leakage**: → `HIDE_EDITOR_FROM_DATA_CONTEXT_PROPERTY` + key consuming + - **Transparency/bleeding**: → Opaque backgrounds, `clipRect`, component fills inlay bounds + - **Threading crashes**: → Single `WriteIntentReadAction.compute{}` snapshots for ALL editor reads + - **Cut buttons**: → Dynamic `getPreferredSize()` (Swing computes natural height) + - **Invisible borders**: → Manual `paintBorder()` with `ColorUtil.mix(bgColor, fg, 0.22)` + alpha + - **Multiple panels**: → Single inlay at `selectionStart` (multi-caret safe) + +5. **Pending Tasks:** + - **None explicit**. LSP endpoint confirmed (`/v1/code-edit`). Feature visually complete. + +6. **Current Work:** + Implemented **guaranteed-visible border** via manual `paintBorder()` override in `InlineEditInputComponent.kt`. User confirmed "almost superb!" then asked "print what endpoint is supposed to be". Confirmed `POST /v1/code-edit` with request/response format. Build successful. + +7. **Optional Next Step:** + Test LSP integration with `./gradlew runIde` + verify `/v1/code-edit` returns valid `modified_code`. + +8. **Direct Quotes for Next Step Context:** + - User: "It almost superb! The one thing - borders are invisible!" → Fixed with manual border painting + - User: "print what endpoint is supposed to be" → Confirmed `/v1/code-edit` + + +Please, continue the conversation based on the provided summary diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..e69de29bb diff --git a/refact-agent/engine/src/at_commands/at_knowledge.rs b/refact-agent/engine/src/at_commands/at_knowledge.rs index 551872c09..f7001f844 100644 --- a/refact-agent/engine/src/at_commands/at_knowledge.rs +++ b/refact-agent/engine/src/at_commands/at_knowledge.rs @@ -9,16 +9,13 @@ use crate::at_commands::execute_at::AtCommandMember; use crate::call_validation::{ChatMessage, ContextEnum}; use crate::memories::memories_search; -/// @knowledge-load command - loads knowledge entries by search key or memory ID pub struct AtLoadKnowledge { params: Vec>, } impl AtLoadKnowledge { pub fn new() -> Self { - AtLoadKnowledge { - params: vec![], - } + AtLoadKnowledge { params: vec![] } } } @@ -38,31 +35,37 @@ impl AtCommand for AtLoadKnowledge { return Err("Usage: @knowledge-load ".to_string()); } - let search_key = args.iter().map(|x| x.text.clone()).join(" ").to_string(); - let gcx = { - let ccx_locked = ccx.lock().await; - ccx_locked.global_context.clone() - }; + let search_key = args.iter().map(|x| x.text.clone()).join(" "); + let gcx = ccx.lock().await.global_context.clone(); - let mem_top_n = 5; - let memories = memories_search(gcx.clone(), &search_key, mem_top_n).await?; + let memories = memories_search(gcx, &search_key, 5).await?; let mut seen_memids = HashSet::new(); let unique_memories: Vec<_> = memories.into_iter() - .filter(|m| seen_memids.insert(m.iknow_id.clone())) + .filter(|m| seen_memids.insert(m.memid.clone())) .collect(); - let mut results = String::new(); - for memory in unique_memories { - let mut content = String::new(); - content.push_str(&format!("🗃️{}\n", memory.iknow_id)); - content.push_str(&memory.iknow_memory); - results.push_str(&content); - }; + + let results = unique_memories.iter().map(|m| { + let mut result = String::new(); + if let Some(path) = &m.file_path { + result.push_str(&format!("📄 {}", path.display())); + if let Some((start, end)) = m.line_range { + result.push_str(&format!(":{}-{}", start, end)); + } + result.push('\n'); + } + if let Some(title) = &m.title { + result.push_str(&format!("📌 {}\n", title)); + } + result.push_str(&m.content); + result.push_str("\n\n"); + result + }).collect::(); let context = ContextEnum::ChatMessage(ChatMessage::new("plain_text".to_string(), results)); Ok((vec![context], "".to_string())) } fn depends_on(&self) -> Vec { - vec!["knowledge".to_string()] + vec![] } } diff --git a/refact-agent/engine/src/background_tasks.rs b/refact-agent/engine/src/background_tasks.rs index 4e871b3ec..18a821718 100644 --- a/refact-agent/engine/src/background_tasks.rs +++ b/refact-agent/engine/src/background_tasks.rs @@ -39,16 +39,14 @@ impl BackgroundTasksHolder { } } -pub async fn start_background_tasks(gcx: Arc>, config_dir: &PathBuf) -> BackgroundTasksHolder { +pub async fn start_background_tasks(gcx: Arc>, _config_dir: &PathBuf) -> BackgroundTasksHolder { let mut bg = BackgroundTasksHolder::new(vec![ tokio::spawn(crate::files_in_workspace::files_in_workspace_init_task(gcx.clone())), tokio::spawn(crate::telemetry::basic_transmit::telemetry_background_task(gcx.clone())), tokio::spawn(crate::snippets_transmit::tele_snip_background_task(gcx.clone())), - tokio::spawn(crate::vecdb::vdb_highlev::vecdb_background_reload(gcx.clone())), // this in turn can create global_context::vec_db + tokio::spawn(crate::vecdb::vdb_highlev::vecdb_background_reload(gcx.clone())), tokio::spawn(crate::integrations::sessions::remove_expired_sessions_background_task(gcx.clone())), - tokio::spawn(crate::memories::memories_migration(gcx.clone(), config_dir.clone())), tokio::spawn(crate::git::cleanup::git_shadow_cleanup_background_task(gcx.clone())), - tokio::spawn(crate::cloud::threads_sub::watch_threads_subscription(gcx.clone())), ]); let ast = gcx.clone().read().await.ast_service.clone(); if let Some(ast_service) = ast { diff --git a/refact-agent/engine/src/cloud/experts_req.rs b/refact-agent/engine/src/cloud/experts_req.rs deleted file mode 100644 index 74fb9ac8b..000000000 --- a/refact-agent/engine/src/cloud/experts_req.rs +++ /dev/null @@ -1,130 +0,0 @@ -use log::error; -use regex::Regex; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use tokio::sync::RwLock as ARwLock; - -use crate::global_context::GlobalContext; - -#[derive(Debug, Serialize, Deserialize)] -pub struct Expert { - pub owner_fuser_id: Option, - pub owner_shared: bool, - pub located_fgroup_id: Option, - pub fexp_name: String, - pub fexp_system_prompt: String, - pub fexp_python_kernel: String, - pub fexp_block_tools: String, - pub fexp_allow_tools: String, -} - -impl Expert { - pub fn is_tool_allowed(&self, tool_name: &str) -> bool { - let mut blocked = false; - if !self.fexp_block_tools.trim().is_empty() { - match Regex::new(&self.fexp_block_tools) { - Ok(re) => { - if re.is_match(tool_name) { - blocked = true; - } - } - Err(e) => { - error!( - "Failed to compile fexp_block_tools regex: {}: {}", - self.fexp_block_tools, e - ); - } - } - } - // Allow if matches allow regex, even if blocked - if !self.fexp_allow_tools.trim().is_empty() { - match Regex::new(&self.fexp_allow_tools) { - Ok(re) => { - if re.is_match(tool_name) { - return true; - } - } - Err(e) => { - error!( - "Failed to compile fexp_allow_tools regex: {}: {}", - self.fexp_allow_tools, e - ); - } - } - } - - !blocked - } -} - -pub async fn get_expert( - gcx: Arc>, - expert_name: &str -) -> Result { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetExpert($id: String!) { - expert_get(id: $id) { - owner_fuser_id - owner_shared - located_fgroup_id - fexp_name - fexp_system_prompt - fexp_python_kernel - fexp_block_tools - fexp_allow_tools - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": { - "id": expert_name - } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(expert_value) = data.get("expert_get") { - let expert: Expert = serde_json::from_value(expert_value.clone()) - .map_err(|e| format!("Failed to parse expert: {}", e))?; - return Ok(expert); - } - } - Err(format!( - "Expert with name '{}' not found or unexpected response format: {}", - expert_name, response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get expert with name {}: HTTP status {}, error: {}", - expert_name, status, error_text - )) - } -} diff --git a/refact-agent/engine/src/cloud/messages_req.rs b/refact-agent/engine/src/cloud/messages_req.rs deleted file mode 100644 index e1c6a9bb0..000000000 --- a/refact-agent/engine/src/cloud/messages_req.rs +++ /dev/null @@ -1,365 +0,0 @@ -use crate::call_validation::{ChatContent, ChatMessage, ChatToolCall, ChatUsage, DiffChunk}; -use crate::global_context::GlobalContext; -use log::error; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use itertools::Itertools; -use tokio::sync::RwLock as ARwLock; -use tracing::warn; - -#[derive(Debug, Serialize, Deserialize)] -pub struct ThreadMessage { - pub ftm_belongs_to_ft_id: String, - pub ftm_alt: i32, - pub ftm_num: i32, - pub ftm_prev_alt: i32, - pub ftm_role: String, - pub ftm_content: Option, - pub ftm_tool_calls: Option, - pub ftm_call_id: String, - pub ftm_usage: Option, - pub ftm_created_ts: f64, - pub ftm_provenance: Value, -} - -pub async fn get_thread_messages( - gcx: Arc>, - thread_id: &str, - alt: i64, -) -> Result, String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetThreadMessagesByAlt($thread_id: String!, $alt: Int!) { - thread_messages_list( - ft_id: $thread_id, - ftm_alt: $alt - ) { - ftm_belongs_to_ft_id - ftm_alt - ftm_num - ftm_prev_alt - ftm_role - ftm_content - ftm_tool_calls - ftm_call_id - ftm_usage - ftm_created_ts - ftm_provenance - } - } - "#; - let variables = json!({ - "thread_id": thread_id, - "alt": alt - }); - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": variables - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(messages) = data.get("thread_messages_list") { - let messages: Vec = serde_json::from_value(messages.clone()) - .map_err(|e| format!("Failed to parse thread messages: {}", e))?; - return Ok(messages); - } - } - Err(format!("Unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread messages: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn create_thread_messages( - gcx: Arc>, - thread_id: &str, - messages: Vec, -) -> Result<(), String> { - if messages.is_empty() { - return Err("No messages provided".to_string()); - } - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let mut input_messages = Vec::with_capacity(messages.len()); - for message in messages { - if message.ftm_belongs_to_ft_id != thread_id { - return Err(format!( - "Message thread ID {} doesn't match the provided thread ID {}", - message.ftm_belongs_to_ft_id, thread_id - )); - } - if message.ftm_role.is_empty() { - return Err("Message role is required".to_string()); - } - let tool_calls_str = match &message.ftm_tool_calls { - Some(tc) => serde_json::to_string(tc) - .map_err(|e| format!("Failed to serialize tool_calls: {}", e))?, - None => "{}".to_string(), - }; - let usage_str = match &message.ftm_usage { - Some(u) => { - serde_json::to_string(u).map_err(|e| format!("Failed to serialize usage: {}", e))? - } - None => "{}".to_string(), - }; - input_messages.push(json!({ - "ftm_belongs_to_ft_id": message.ftm_belongs_to_ft_id, - "ftm_alt": message.ftm_alt, - "ftm_num": message.ftm_num, - "ftm_prev_alt": message.ftm_prev_alt, - "ftm_role": message.ftm_role, - "ftm_content": serde_json::to_string(&message.ftm_content).unwrap(), - "ftm_tool_calls": tool_calls_str, - "ftm_call_id": message.ftm_call_id, - "ftm_usage": usage_str, - "ftm_provenance": serde_json::to_string(&message.ftm_provenance).unwrap() - })); - } - let variables = json!({ - "input": { - "ftm_belongs_to_ft_id": thread_id, - "messages": input_messages - } - }); - let mutation = r#" - mutation ThreadMessagesCreateMultiple($input: FThreadMultipleMessagesInput!) { - thread_messages_create_multiple(input: $input) { - count - messages { - ftm_belongs_to_ft_id - ftm_alt - ftm_num - ftm_prev_alt - ftm_role - ftm_created_ts - ftm_call_id - ftm_provenance - } - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": mutation, - "variables": variables - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(_) = response_json.get("data") { - return Ok(()) - } - Err(format!("Unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to create thread messages: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub fn convert_thread_messages_to_messages( - thread_messages: &Vec, -) -> Vec { - thread_messages - .iter() - .map(|msg| { - let content: ChatContent = if let Some(content) = &msg.ftm_content { - serde_json::from_value(content.clone()).unwrap_or_default() - } else { - ChatContent::default() - }; - let tool_calls = msg.ftm_tool_calls.clone().map(|tc| { - serde_json::from_value::>(tc).unwrap_or_else(|_| vec![]) - }); - ChatMessage { - role: msg.ftm_role.clone(), - content, - tool_calls, - tool_call_id: msg.ftm_call_id.clone(), - tool_failed: None, - usage: msg.ftm_usage.clone().map(|u| { - serde_json::from_value::(u).unwrap_or_else(|_| ChatUsage::default()) - }), - checkpoints: vec![], - thinking_blocks: None, - finish_reason: None, - output_filter: None, - } - }) - .collect() -} - -pub fn convert_messages_to_thread_messages( - messages: Vec, - alt: i32, - prev_alt: i32, - start_num: i32, - thread_id: &str, -) -> Result, String> { - let mut output_messages = Vec::new(); - let flush_delayed_images = |results: &mut Vec, delay_images: &mut Vec| { - results.extend(delay_images.clone()); - delay_images.clear(); - }; - for (i, msg) in messages.into_iter().enumerate() { - let num = start_num + i as i32; - let mut delay_images = vec![]; - let mut messages = if msg.role == "tool" { - let mut results = vec![]; - match &msg.content { - ChatContent::Multimodal(multimodal_content) => { - let texts = multimodal_content.iter().filter(|x|x.is_text()).collect::>(); - let images = multimodal_content.iter().filter(|x|x.is_image()).collect::>(); - let text = if texts.is_empty() { - "attached images below".to_string() - } else { - texts.iter().map(|x|x.m_content.clone()).collect::>().join("\n") - }; - let mut msg_cloned = msg.clone(); - msg_cloned.content = ChatContent::SimpleText(text); - results.push(msg_cloned.into_value(&None, "")); - if !images.is_empty() { - let msg_img = ChatMessage { - role: "user".to_string(), - content: ChatContent::Multimodal(images.into_iter().cloned().collect()), - ..Default::default() - }; - delay_images.push(msg_img.into_value(&None, "")); - } - }, - ChatContent::SimpleText(_) => { - results.push(msg.into_value(&None, "")); - } - } - results - } else if msg.role == "assistant" || msg.role == "system" { - vec![msg.into_value(&None, "")] - } else if msg.role == "user" { - vec![msg.into_value(&None, "")] - } else if msg.role == "diff" { - let extra_message = match serde_json::from_str::>(&msg.content.content_text_only()) { - Ok(chunks) => { - if chunks.is_empty() { - "Nothing has changed.".to_string() - } else { - chunks.iter() - .filter(|x| !x.application_details.is_empty()) - .map(|x| x.application_details.clone()) - .join("\n") - } - }, - Err(_) => "".to_string() - }; - vec![ChatMessage { - role: "diff".to_string(), - content: ChatContent::SimpleText(format!("The operation has succeeded.\n{extra_message}")), - tool_calls: None, - tool_call_id: msg.tool_call_id.clone(), - ..Default::default() - }.into_value(&None, "")] - } else if msg.role == "plain_text" || msg.role == "cd_instruction" || msg.role == "context_file" { - vec![ChatMessage::new( - msg.role.to_string(), - msg.content.content_text_only(), - ).into_value(&None, "")] - } else { - warn!("unknown role: {}", msg.role); - vec![] - }; - flush_delayed_images(&mut messages, &mut delay_images); - for pp_msg in messages { - let tool_calls = pp_msg.get("tool_calls") - .map(|x| Some(x.clone())).unwrap_or(None); - let usage = pp_msg.get("usage") - .map(|x| Some(x.clone())).unwrap_or(None); - let content = pp_msg - .get("content") - .cloned() - .ok_or("cannot find role in the message".to_string())?; - output_messages.push(ThreadMessage { - ftm_belongs_to_ft_id: thread_id.to_string(), - ftm_alt: alt, - ftm_num: num, - ftm_prev_alt: prev_alt, - ftm_role: msg.role.clone(), - ftm_content: Some(content), - ftm_tool_calls: tool_calls, - ftm_call_id: msg.tool_call_id.clone(), - ftm_usage: usage, - ftm_created_ts: std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_secs_f64(), - ftm_provenance: json!({"system_type": "refact_lsp", "version": env!("CARGO_PKG_VERSION") }), - }) - } - } - Ok(output_messages) -} - -pub async fn get_tool_names_from_openai_format( - toolset_json: &Vec, -) -> Result, String> { - let mut tool_names = Vec::new(); - for tool in toolset_json { - if let Some(function) = tool.get("function") { - if let Some(name) = function.get("name") { - if let Some(name_str) = name.as_str() { - tool_names.push(name_str.to_string()); - } - } - } - } - Ok(tool_names) -} diff --git a/refact-agent/engine/src/cloud/mod.rs b/refact-agent/engine/src/cloud/mod.rs deleted file mode 100644 index e24f2d519..000000000 --- a/refact-agent/engine/src/cloud/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod threads_sub; -mod threads_req; -mod messages_req; -mod experts_req; diff --git a/refact-agent/engine/src/cloud/threads_req.rs b/refact-agent/engine/src/cloud/threads_req.rs deleted file mode 100644 index fbd74d4b9..000000000 --- a/refact-agent/engine/src/cloud/threads_req.rs +++ /dev/null @@ -1,300 +0,0 @@ -use log::error; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use tokio::sync::RwLock as ARwLock; - -use crate::global_context::GlobalContext; - -#[derive(Debug, Serialize, Deserialize)] -pub struct Thread { - pub owner_fuser_id: String, - pub owner_shared: bool, - pub located_fgroup_id: String, - pub ft_id: String, - pub ft_fexp_name: String, - pub ft_title: String, - pub ft_toolset: Vec, - pub ft_belongs_to_fce_id: Option, - pub ft_model: String, - pub ft_temperature: f64, - pub ft_max_new_tokens: i32, - pub ft_n: i32, - pub ft_error: Option, - pub ft_need_assistant: i32, - pub ft_need_tool_calls: i32, - pub ft_anything_new: bool, - pub ft_created_ts: f64, - pub ft_updated_ts: f64, - pub ft_archived_ts: f64, - pub ft_locked_by: String, -} - -pub async fn get_thread( - gcx: Arc>, - thread_id: &str, -) -> Result { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetThread($id: String!) { - thread_get(id: $id) { - owner_fuser_id - owner_shared - located_fgroup_id - ft_id - ft_fexp_name, - ft_title - ft_belongs_to_fce_id - ft_model - ft_temperature - ft_max_new_tokens - ft_n - ft_error - ft_toolset - ft_need_assistant - ft_need_tool_calls - ft_anything_new - ft_created_ts - ft_updated_ts - ft_archived_ts - ft_locked_by - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": {"id": thread_id} - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(thread_value) = data.get("thread_get") { - let thread: Thread = serde_json::from_value(thread_value.clone()) - .map_err(|e| format!("Failed to parse thread: {}", e))?; - return Ok(thread); - } - } - Err(format!( - "Thread not found or unexpected response format: {}", - response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn set_thread_toolset( - gcx: Arc>, - thread_id: &str, - ft_toolset: Vec, -) -> Result, String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let mutation = r#" - mutation UpdateThread($thread_id: String!, $patch: FThreadPatch!) { - thread_patch(id: $thread_id, patch: $patch) { - ft_toolset - } - } - "#; - let variables = json!({ - "thread_id": thread_id, - "patch": { - "ft_toolset": serde_json::to_string(&ft_toolset).unwrap() - } - }); - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": mutation, - "variables": variables - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(ft_toolset_json) = data.get("thread_patch") { - let ft_toolset: Vec = - serde_json::from_value(ft_toolset_json["ft_toolset"].clone()) - .map_err(|e| format!("Failed to parse updated thread: {}", e))?; - return Ok(ft_toolset); - } - } - Err(format!("Unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to update thread: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn lock_thread( - gcx: Arc>, - thread_id: &str, - hash: &str, -) -> Result<(), String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let worker_name = format!("refact-lsp:{hash}"); - let query = r#" - mutation AdvanceLock($ft_id: String!, $worker_name: String!) { - thread_lock(ft_id: $ft_id, worker_name: $worker_name) - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": {"ft_id": thread_id, "worker_name": worker_name} - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if data.get("thread_lock").is_some() { - return Ok(()); - } else { - return Err(format!("Thread {thread_id} is locked by another worker")); - } - } - Err(format!( - "Thread not found or unexpected response format: {}", - response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread: HTTP status {}, error: {}", - status, error_text - )) - } -} - -pub async fn unlock_thread( - gcx: Arc>, - thread_id: String, - hash: String, -) -> Result<(), String> { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let worker_name = format!("refact-lsp:{hash}"); - let query = r#" - mutation AdvanceUnlock($ft_id: String!, $worker_name: String!) { - thread_unlock(ft_id: $ft_id, worker_name: $worker_name) - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({ - "query": query, - "variables": {"ft_id": thread_id, "worker_name": worker_name} - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if data.get("thread_unlock").is_some() { - return Ok(()); - } else { - return Err(format!("Cannot unlock thread {thread_id}")); - } - } - Err(format!( - "Thread not found or unexpected response format: {}", - response_body - )) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get thread: HTTP status {}, error: {}", - status, error_text - )) - } -} diff --git a/refact-agent/engine/src/cloud/threads_sub.rs b/refact-agent/engine/src/cloud/threads_sub.rs deleted file mode 100644 index f15998e48..000000000 --- a/refact-agent/engine/src/cloud/threads_sub.rs +++ /dev/null @@ -1,512 +0,0 @@ -use std::collections::HashSet; -use crate::global_context::GlobalContext; -use futures::{SinkExt, StreamExt}; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::sync::Arc; -use std::sync::atomic::Ordering; -use std::time::Duration; -use indexmap::IndexMap; -use tokio::sync::RwLock as ARwLock; -use tokio::sync::Mutex as AMutex; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::{connect_async, tungstenite::protocol::Message}; -use tracing::{error, info, warn}; -use url::Url; -use crate::at_commands::at_commands::AtCommandsContext; -use crate::call_validation::ChatContent; -use crate::cloud::messages_req::ThreadMessage; -use crate::cloud::threads_req::{lock_thread, Thread}; -use rand::{Rng, thread_rng}; -use rand::distributions::Alphanumeric; -use crate::custom_error::MapErrToString; - - -const RECONNECT_DELAY_SECONDS: u64 = 3; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ThreadPayload { - pub owner_fuser_id: String, - pub ft_id: String, - pub ft_error: Option, - pub ft_locked_by: String, - pub ft_need_tool_calls: i64, - pub ft_app_searchable: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct BasicStuff { - pub fuser_id: String, - pub workspaces: Vec, -} - -const THREADS_SUBSCRIPTION_QUERY: &str = r#" - subscription ThreadsPageSubs($located_fgroup_id: String!) { - threads_in_group(located_fgroup_id: $located_fgroup_id) { - news_action - news_payload_id - news_payload { - owner_fuser_id - ft_id - ft_error - ft_locked_by - ft_need_tool_calls - ft_app_searchable - } - } - } -"#; - -pub async fn trigger_threads_subscription_restart(gcx: Arc>) { - let restart_flag = gcx.read().await.threads_subscription_restart_flag.clone(); - restart_flag.store(true, Ordering::SeqCst); - info!("threads subscription restart triggered"); -} - -pub async fn watch_threads_subscription(gcx: Arc>) { - if !gcx.read().await.cmdline.cloud_threads { - return; - } - - loop { - { - let restart_flag = gcx.read().await.threads_subscription_restart_flag.clone(); - restart_flag.store(false, Ordering::SeqCst); - } - let located_fgroup_id = if let Some(located_fgroup_id) = gcx.read().await.active_group_id.clone() { - located_fgroup_id - } else { - warn!("no active group is set, skipping threads subscription"); - tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - continue; - }; - - info!( - "starting subscription for threads_in_group with fgroup_id=\"{}\"", - located_fgroup_id - ); - let connection_result = initialize_connection(gcx.clone()).await; - let mut connection = match connection_result { - Ok(conn) => conn, - Err(err) => { - error!("failed to initialize connection: {}", err); - info!("will attempt to reconnect in {} seconds", RECONNECT_DELAY_SECONDS); - tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - continue; - } - }; - - let events_result = events_loop(gcx.clone(), &mut connection).await; - if let Err(err) = events_result { - error!("failed to process events: {}", err); - info!("will attempt to reconnect in {} seconds", RECONNECT_DELAY_SECONDS); - } - - if gcx.read().await.shutdown_flag.load(Ordering::SeqCst) { - info!("shutting down threads subscription"); - break; - } - - let restart_flag = gcx.read().await.threads_subscription_restart_flag.clone(); - if !restart_flag.load(Ordering::SeqCst) { - tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - } - } -} - -async fn initialize_connection(gcx: Arc>) -> Result< - futures::stream::SplitStream< - tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream - >, - >, - String, -> { - let (api_key, located_fgroup_id) = { - let gcx_read = gcx.read().await; - (gcx_read.cmdline.api_key.clone(), - gcx_read.active_group_id.clone().unwrap_or_default()) - }; - let url = Url::parse(crate::constants::GRAPHQL_WS_URL) - .map_err(|e| format!("Failed to parse WebSocket URL: {}", e))?; - let mut request = url - .into_client_request() - .map_err(|e| format!("Failed to create request: {}", e))?; - request - .headers_mut() - .insert("Sec-WebSocket-Protocol", "graphql-ws".parse().unwrap()); - let (ws_stream, _) = connect_async(request) - .await - .map_err(|e| format!("Failed to connect to WebSocket server: {}", e))?; - let (mut write, mut read) = ws_stream.split(); - let init_message = json!({ - "type": "connection_init", - "payload": { - "apikey": api_key - } - }); - write.send(Message::Text(init_message.to_string())).await - .map_err(|e| format!("Failed to send connection init message: {}", e))?; - - let timeout = tokio::time::timeout(Duration::from_secs(5), read.next()) - .await - .map_err(|_| "Timeout waiting for connection acknowledgment".to_string())?; - - if let Some(msg) = timeout { - let msg = msg.map_err(|e| format!("WebSocket error: {}", e))?; - match msg { - Message::Text(text) => { - info!("Received response: {}", text); - let response: Value = serde_json::from_str(&text) - .map_err(|e| format!("Failed to parse connection response: {}", e))?; - if let Some(msg_type) = response["type"].as_str() { - if msg_type == "connection_ack" { - } else if msg_type == "connection_error" { - return Err(format!("Connection error: {}", response)); - } else { - return Err(format!("Expected connection_ack, got: {}", response)); - } - } else { - return Err(format!( - "Invalid response format, missing 'type': {}", - response - )); - } - } - Message::Close(frame) => { - return if let Some(frame) = frame { - Err(format!( - "WebSocket closed during initialization: code={}, reason={}", - frame.code, frame.reason - )) - } else { - Err( - "WebSocket connection closed during initialization without details" - .to_string(), - ) - } - } - _ => { - return Err(format!("Unexpected message type received: {:?}", msg)); - } - } - } else { - return Err("No response received for connection initialization".to_string()); - } - let subscription_message = json!({ - "id": "42", - "type": "start", - "payload": { - "query": THREADS_SUBSCRIPTION_QUERY, - "variables": { - "located_fgroup_id": located_fgroup_id - } - } - }); - write - .send(Message::Text(subscription_message.to_string())) - .await - .map_err(|e| format!("Failed to send subscription message: {}", e))?; - Ok(read) -} - -async fn events_loop( - gcx: Arc>, - connection: &mut futures::stream::SplitStream< - tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, - >, -) -> Result<(), String> { - info!("cloud threads subscription started, waiting for events..."); - let basic_info = get_basic_info(gcx.clone()).await?; - while let Some(msg) = connection.next().await { - if gcx.read().await.shutdown_flag.load(Ordering::SeqCst) { - info!("shutting down threads subscription"); - break; - } - if gcx.read().await.threads_subscription_restart_flag.load(Ordering::SeqCst) { - info!("restart flag detected, restarting threads subscription"); - return Ok(()); - } - match msg { - Ok(Message::Text(text)) => { - let response: Value = match serde_json::from_str(&text) { - Ok(res) => res, - Err(err) => { - error!("failed to parse message: {}, error: {}", text, err); - continue; - } - }; - let response_type = response["type"].as_str().unwrap_or("unknown"); - match response_type { - "data" => { - if let Some(payload) = response["payload"].as_object() { - let data = &payload["data"]; - let threads_in_group = &data["threads_in_group"]; - let news_action = threads_in_group["news_action"].as_str().unwrap_or(""); - if news_action != "INSERT" && news_action != "UPDATE" { - continue; - } - if let Ok(payload) = serde_json::from_value::(threads_in_group["news_payload"].clone()) { - match process_thread_event(gcx.clone(), &payload, &basic_info).await { - Ok(_) => {} - Err(err) => { - error!("failed to process thread event: {}", err); - } - } - } else { - info!("failed to parse thread payload: {}", text); - } - } else { - info!("received data message but couldn't find payload"); - } - } - "error" => { - error!("threads subscription error: {}", text); - } - _ => { - info!("received message with unknown type: {}", text); - } - } - } - Ok(Message::Close(_)) => { - info!("webSocket connection closed"); - break; - } - Ok(_) => {} - Err(e) => { - return Err(format!("webSocket error: {}", e)); - } - } - } - Ok(()) -} -fn generate_random_hash(length: usize) -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(length) - .map(char::from) - .collect() -} - -async fn get_basic_info(gcx: Arc>) -> Result { - let client = Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let query = r#" - query GetBasicInfo { - query_basic_stuff { - fuser_id - workspaces { - ws_id - ws_owner_fuser_id - ws_root_group_id - root_group_name - } - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .json(&json!({"query": query})) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - return Err(format!("GraphQL request error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - let basic_stuff_struct: BasicStuff = serde_json::from_value(data["query_basic_stuff"].clone()) - .map_err(|e| format!("Failed to parse updated thread: {}", e))?; - return Ok(basic_stuff_struct); - } - Err(format!("Basic data not found or unexpected response format: {}", response_body)) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!( - "Failed to get basic data: HTTP status {}, error: {}", - status, error_text - )) - } -} - -async fn process_thread_event( - gcx: Arc>, - thread_payload: &ThreadPayload, - basic_info: &BasicStuff -) -> Result<(), String> { - if thread_payload.ft_need_tool_calls == -1 || thread_payload.owner_fuser_id != basic_info.fuser_id { - return Ok(()); - } - let app_searchable_id = gcx.read().await.app_searchable_id.clone(); - if let Some(ft_app_searchable) = thread_payload.ft_app_searchable.clone() { - if ft_app_searchable != app_searchable_id { - info!("thread `{}` has different `app_searchable` id, skipping it", thread_payload.ft_id); - } - } else { - info!("thread `{}` doesn't have the `app_searchable` id, skipping it", thread_payload.ft_id); - return Ok(()); - } - if let Some(error) = thread_payload.ft_error.as_ref() { - info!("thread `{}` has the error: `{}`. Skipping it", thread_payload.ft_id, error); - return Ok(()); - } - let messages = crate::cloud::messages_req::get_thread_messages( - gcx.clone(), - &thread_payload.ft_id, - thread_payload.ft_need_tool_calls, - ).await?; - if messages.is_empty() { - info!("thread `{}` has no messages. Skipping it", thread_payload.ft_id); - return Ok(()); - } - let thread = crate::cloud::threads_req::get_thread(gcx.clone(), &thread_payload.ft_id).await?; - let hash = generate_random_hash(16); - match lock_thread(gcx.clone(), &thread.ft_id, &hash).await { - Ok(_) => {} - Err(err) => return Err(err) - } - let result = if messages.iter().all(|x| x.ftm_role != "system") { - initialize_thread(gcx.clone(), &thread.ft_fexp_name, &thread, &messages).await - } else { - call_tools(gcx.clone(), &thread, &messages).await - }; - match crate::cloud::threads_req::unlock_thread(gcx.clone(), thread.ft_id.clone(), hash).await { - Ok(_) => info!("thread `{}` unlocked successfully", thread.ft_id), - Err(err) => error!("failed to unlock thread `{}`: {}", thread.ft_id, err), - } - result -} - -async fn initialize_thread( - gcx: Arc>, - expert_name: &str, - thread: &Thread, - thread_messages: &Vec, -) -> Result<(), String> { - let expert = crate::cloud::experts_req::get_expert(gcx.clone(), expert_name).await?; - let tools: Vec> = - crate::tools::tools_list::get_available_tools(gcx.clone()) - .await - .into_iter() - .filter(|tool| expert.is_tool_allowed(&tool.tool_description().name)) - .collect(); - let tool_descriptions = tools - .iter() - .map(|x| x.tool_description().into_openai_style()) - .collect::>(); - crate::cloud::threads_req::set_thread_toolset(gcx.clone(), &thread.ft_id, tool_descriptions).await?; - let updated_system_prompt = crate::scratchpads::chat_utils_prompts::system_prompt_add_extra_instructions( - gcx.clone(), expert.fexp_system_prompt.clone(), HashSet::new(), &crate::call_validation::ChatMeta::default() - ).await; - let last_message = thread_messages.last().unwrap(); - let output_thread_messages = vec![ThreadMessage { - ftm_belongs_to_ft_id: last_message.ftm_belongs_to_ft_id.clone(), - ftm_alt: last_message.ftm_alt.clone(), - ftm_num: 0, - ftm_prev_alt: 100, - ftm_role: "system".to_string(), - ftm_content: Some( - serde_json::to_value(ChatContent::SimpleText(updated_system_prompt)).unwrap(), - ), - ftm_tool_calls: None, - ftm_call_id: "".to_string(), - ftm_usage: None, - ftm_created_ts: std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs_f64(), - ftm_provenance: json!({"important": "information"}), - }]; - crate::cloud::messages_req::create_thread_messages( - gcx.clone(), - &thread.ft_id, - output_thread_messages, - ).await?; - Ok(()) -} - -async fn call_tools( - gcx: Arc>, - thread: &Thread, - thread_messages: &Vec, -) -> Result<(), String> { - let max_new_tokens = 8192; - let last_message_num = thread_messages.iter().map(|x| x.ftm_num).max().unwrap_or(0); - let (alt, prev_alt) = thread_messages - .last() - .map(|msg| (msg.ftm_alt, msg.ftm_prev_alt)) - .unwrap_or((0, 0)); - let messages = crate::cloud::messages_req::convert_thread_messages_to_messages(thread_messages); - let caps = crate::global_context::try_load_caps_quickly_if_not_present(gcx.clone(), 0) - .await - .map_err_to_string()?; - let model_rec = crate::caps::resolve_chat_model(caps, &format!("refact/{}", thread.ft_model)) - .map_err(|e| format!("Failed to resolve chat model: {}", e))?; - let ccx = Arc::new(AMutex::new( - AtCommandsContext::new( - gcx.clone(), - model_rec.base.n_ctx, - 12, - false, - messages.clone(), - thread.ft_id.to_string(), - false, - thread.ft_model.to_string(), - ).await, - )); - let allowed_tools = crate::cloud::messages_req::get_tool_names_from_openai_format(&thread.ft_toolset).await?; - let mut all_tools: IndexMap> = - crate::tools::tools_list::get_available_tools(gcx.clone()).await - .into_iter() - .filter(|x| allowed_tools.contains(&x.tool_description().name)) - .map(|x| (x.tool_description().name, x)) - .collect(); - let mut has_rag_results = crate::scratchpads::scratchpad_utils::HasRagResults::new(); - let tokenizer_arc = crate::tokens::cached_tokenizer(gcx.clone(), &model_rec.base).await?; - let messages_count = messages.len(); - let (output_messages, _) = crate::tools::tools_execute::run_tools_locally( - ccx.clone(), - &mut all_tools, - tokenizer_arc, - max_new_tokens, - &messages, - &mut has_rag_results, - &None, - ).await?; - if messages.len() == output_messages.len() { - tracing::warn!( - "Thread has no active tool call awaiting but still has need_tool_call turned on" - ); - return Ok(()); - } - let output_thread_messages = crate::cloud::messages_req::convert_messages_to_thread_messages( - output_messages.into_iter().skip(messages_count).collect(), - alt, - prev_alt, - last_message_num + 1, - &thread.ft_id, - )?; - crate::cloud::messages_req::create_thread_messages( - gcx.clone(), - &thread.ft_id, - output_thread_messages, - ).await?; - Ok(()) -} diff --git a/refact-agent/engine/src/constants.rs b/refact-agent/engine/src/constants.rs index 248ad7bcf..aff197fc6 100644 --- a/refact-agent/engine/src/constants.rs +++ b/refact-agent/engine/src/constants.rs @@ -1,3 +1 @@ pub const CLOUD_URL: &str = "https://flexus.team/v1"; -pub const GRAPHQL_WS_URL: &str = "ws://flexus.team/v1/graphql"; -pub const GRAPHQL_URL: &str = "https://flexus.team/v1/graphql"; diff --git a/refact-agent/engine/src/file_filter.rs b/refact-agent/engine/src/file_filter.rs index 4b75075c8..46539bd07 100644 --- a/refact-agent/engine/src/file_filter.rs +++ b/refact-agent/engine/src/file_filter.rs @@ -6,6 +6,10 @@ use std::path::PathBuf; const LARGE_FILE_SIZE_THRESHOLD: u64 = 4096*1024; // 4Mb files const SMALL_FILE_SIZE_THRESHOLD: u64 = 5; // 5 Bytes +pub const KNOWLEDGE_FOLDER_NAME: &str = ".refact_knowledge"; + +const ALLOWED_HIDDEN_FOLDERS: &[&str] = &[KNOWLEDGE_FOLDER_NAME]; + pub const SOURCE_FILE_EXTENSIONS: &[&str] = &[ "c", "cpp", "cc", "h", "hpp", "cs", "java", "py", "rb", "go", "rs", "swift", "php", "js", "jsx", "ts", "tsx", "lua", "pl", "r", "sh", "bat", "cmd", "ps1", @@ -16,12 +20,22 @@ pub const SOURCE_FILE_EXTENSIONS: &[&str] = &[ "gradle", "liquid" ]; +fn is_in_allowed_hidden_folder(path: &PathBuf) -> bool { + path.ancestors().any(|ancestor| { + ancestor.file_name() + .map(|name| ALLOWED_HIDDEN_FOLDERS.contains(&name.to_string_lossy().as_ref())) + .unwrap_or(false) + }) +} + pub fn is_valid_file(path: &PathBuf, allow_hidden_folders: bool, ignore_size_thresholds: bool) -> Result<(), Box> { if !path.is_file() { return Err("Path is not a file".into()); } - if !allow_hidden_folders && path.ancestors().any(|ancestor| { + let in_allowed_hidden = is_in_allowed_hidden_folder(path); + + if !allow_hidden_folders && !in_allowed_hidden && path.ancestors().any(|ancestor| { ancestor.file_name() .map(|name| name.to_string_lossy().starts_with('.')) .unwrap_or(false) diff --git a/refact-agent/engine/src/files_in_workspace.rs b/refact-agent/engine/src/files_in_workspace.rs index 04752edc1..6c96fbc99 100644 --- a/refact-agent/engine/src/files_in_workspace.rs +++ b/refact-agent/engine/src/files_in_workspace.rs @@ -658,7 +658,6 @@ pub async fn on_workspaces_init(gcx: Arc>) -> i32 let new_app_searchable_id = get_app_searchable_id(&folders); if old_app_searchable_id != new_app_searchable_id { gcx.write().await.app_searchable_id = get_app_searchable_id(&folders); - crate::cloud::threads_sub::trigger_threads_subscription_restart(gcx.clone()).await; } watcher_init(gcx.clone()).await; let files_enqueued = enqueue_all_files_from_workspace_folders(gcx.clone(), false, false).await; diff --git a/refact-agent/engine/src/global_context.rs b/refact-agent/engine/src/global_context.rs index 4b6797640..c3acd17be 100644 --- a/refact-agent/engine/src/global_context.rs +++ b/refact-agent/engine/src/global_context.rs @@ -107,8 +107,6 @@ pub struct CommandLine { #[structopt(long, help="An pre-setup active group id")] pub active_group_id: Option, - #[structopt(long, help="Enable cloud threads support")] - pub cloud_threads: bool, } impl CommandLine { @@ -180,7 +178,6 @@ pub struct GlobalContext { pub init_shadow_repos_lock: Arc>, pub git_operations_abort_flag: Arc, pub app_searchable_id: String, - pub threads_subscription_restart_flag: Arc } pub type SharedGlobalContext = Arc>; // TODO: remove this type alias, confusing @@ -429,7 +426,6 @@ pub async fn create_global_context( init_shadow_repos_lock: Arc::new(AMutex::new(false)), git_operations_abort_flag: Arc::new(AtomicBool::new(false)), app_searchable_id: get_app_searchable_id(&workspace_dirs), - threads_subscription_restart_flag: Arc::new(AtomicBool::new(false)), }; let gcx = Arc::new(ARwLock::new(cx)); crate::files_in_workspace::watcher_init(gcx.clone()).await; diff --git a/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs b/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs index fd20c83c9..4b2a91371 100644 --- a/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs +++ b/refact-agent/engine/src/http/routers/v1/chat_based_handlers.rs @@ -79,24 +79,18 @@ pub async fn handle_v1_trajectory_save( body_bytes: hyper::body::Bytes, ) -> axum::response::Result, ScratchError> { let post = serde_json::from_slice::(&body_bytes).map_err(|e| { - ScratchError::new( - StatusCode::UNPROCESSABLE_ENTITY, - format!("JSON problem: {}", e), - ) + ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, format!("JSON problem: {}", e)) })?; + let trajectory = compress_trajectory(gcx.clone(), &post.messages) .await.map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, e))?; - crate::memories::memories_add( - gcx.clone(), - "trajectory", - &trajectory.as_str(), - false, - ).await.map_err(|e| { - ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("{}", e)) - })?; + + let file_path = crate::memories::save_trajectory(gcx, &trajectory) + .await.map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, e))?; let response = serde_json::json!({ "trajectory": trajectory, + "file_path": file_path.to_string_lossy(), }); Ok(Response::builder() diff --git a/refact-agent/engine/src/http/routers/v1/workspace.rs b/refact-agent/engine/src/http/routers/v1/workspace.rs index 10a13856a..a8f9b964b 100644 --- a/refact-agent/engine/src/http/routers/v1/workspace.rs +++ b/refact-agent/engine/src/http/routers/v1/workspace.rs @@ -23,8 +23,7 @@ pub async fn handle_v1_set_active_group_id( .map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, format!("JSON problem: {}", e)))?; gcx.write().await.active_group_id = Some(post.group_id); - crate::cloud::threads_sub::trigger_threads_subscription_restart(gcx.clone()).await; - + Ok(Response::builder().status(StatusCode::OK).body(Body::from( serde_json::to_string(&serde_json::json!({ "success": true })).unwrap() )).unwrap()) diff --git a/refact-agent/engine/src/main.rs b/refact-agent/engine/src/main.rs index 86fb58f8d..d1f6f4667 100644 --- a/refact-agent/engine/src/main.rs +++ b/refact-agent/engine/src/main.rs @@ -62,7 +62,6 @@ mod http; mod integrations; mod privacy; mod git; -mod cloud; mod agentic; mod memories; mod files_correction_cache; diff --git a/refact-agent/engine/src/memories.rs b/refact-agent/engine/src/memories.rs index 93cea2113..d99b3e59d 100644 --- a/refact-agent/engine/src/memories.rs +++ b/refact-agent/engine/src/memories.rs @@ -1,301 +1,246 @@ use std::path::PathBuf; use std::sync::Arc; -use itertools::Itertools; -use log::error; +use chrono::Local; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; use tokio::sync::RwLock as ARwLock; -use crate::global_context::GlobalContext; use tokio::fs; -use tokio_rusqlite::Connection; -use tracing::{info, warn}; +use tracing::info; +use walkdir::WalkDir; +use crate::file_filter::KNOWLEDGE_FOLDER_NAME; +use crate::files_correction::get_project_dirs; +use crate::files_in_workspace::get_file_text_from_memory_or_disk; +use crate::global_context::GlobalContext; +use crate::vecdb::vdb_markdown_splitter::MarkdownFrontmatter; +use crate::vecdb::vdb_structs::VecdbSearch; #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct MemoRecord { - pub iknow_id: String, - pub iknow_tags: Vec, - pub iknow_memory: String, + pub memid: String, + pub tags: Vec, + pub content: String, + pub file_path: Option, + pub line_range: Option<(u64, u64)>, + pub title: Option, + pub created: Option, } +fn generate_slug(content: &str) -> String { + let first_line = content.lines().next().unwrap_or("knowledge"); + first_line + .chars() + .filter(|c| c.is_alphanumeric() || c.is_whitespace()) + .collect::() + .split_whitespace() + .take(5) + .collect::>() + .join("-") + .to_lowercase() + .chars() + .take(50) + .collect() +} -pub async fn memories_migration( - gcx: Arc>, - config_dir: PathBuf -) { - // Disable migration for now - if true { - return; - } - - if let None = gcx.read().await.active_group_id.clone() { - info!("No active group set up, skipping memory migration"); - return; - } - - let legacy_db_path = config_dir.join("memories.sqlite"); - if !legacy_db_path.exists() { - return; +fn generate_filename(content: &str) -> String { + let timestamp = Local::now().format("%Y-%m-%d_%H%M%S").to_string(); + let slug = generate_slug(content); + if slug.is_empty() { + format!("{}_knowledge.md", timestamp) + } else { + format!("{}_{}.md", timestamp, slug) } - - info!("Found legacy memory database at {:?}, starting migration", legacy_db_path); - - let conn = match Connection::open(&legacy_db_path).await { - Ok(conn) => conn, - Err(e) => { - warn!("Failed to open legacy database: {}", e); - return; - } - }; - - let memories: Vec<(String, String)> = match conn.call(|conn| { - // Query all memories - let mut stmt = conn.prepare("SELECT m_type, m_payload FROM memories")?; - let rows = stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - )) - })?; - - let mut memories = Vec::new(); - for row in rows { - memories.push(row?); - } - - Ok(memories.into_iter().unique_by(|(_, m_payload)| m_payload.clone()).collect()) - }).await { - Ok(memories) => memories, - Err(e) => { - warn!("Failed to query memories: {}", e); - return; - } +} + +fn create_markdown_content(tags: &[String], filenames: &[String], content: &str) -> String { + let now = Local::now().format("%Y-%m-%d").to_string(); + let title = content.lines().next().unwrap_or("Knowledge Entry"); + let tags_str = tags.iter().map(|t| format!("\"{}\"", t)).collect::>().join(", "); + let filenames_str = if filenames.is_empty() { + String::new() + } else { + format!("\nfilenames: [{}]", filenames.iter().map(|f| format!("\"{}\"", f)).collect::>().join(", ")) }; - - if memories.is_empty() { - info!("No memories found in legacy database"); - return; - } - - info!("Found {} memories in legacy database, migrating to cloud", memories.len()); - - // Migrate each memory to the cloud - let mut success_count = 0; - let mut error_count = 0; - for (m_type, m_payload) in memories { - if m_payload.is_empty() { - warn!("Memory payload is empty, skipping"); - continue; - } - match memories_add(gcx.clone(), &m_type, &m_payload, true).await { - Ok(_) => { - success_count += 1; - if success_count % 10 == 0 { - info!("Migrated {} memories so far", success_count); - } - }, - Err(e) => { - error_count += 1; - warn!("Failed to migrate memory: {}", e); - } - } - } - - info!("Memory migration complete: {} succeeded, {} failed", success_count, error_count); - if success_count > 0 { - match fs::remove_file(legacy_db_path.clone()).await { - Ok(_) => info!("Removed legacy database: {:?}", legacy_db_path), - Err(e) => warn!("Failed to remove legacy database: {}", e), - } - } + + format!( + r#"--- +title: "{}" +created: {} +tags: [{}]{} +--- + +{} +"#, + title.trim_start_matches('#').trim(), + now, + tags_str, + filenames_str, + content + ) +} + +async fn get_knowledge_dir(gcx: Arc>) -> Result { + let project_dirs = get_project_dirs(gcx).await; + let workspace_root = project_dirs.first().ok_or("No workspace folder found")?; + Ok(workspace_root.join(KNOWLEDGE_FOLDER_NAME)) } pub async fn memories_add( gcx: Arc>, - m_type: &str, - m_memory: &str, - _unknown_project: bool -) -> Result<(), String> { - let client = reqwest::Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let active_group_id = gcx.read().await.active_group_id.clone() - .ok_or("active_group_id must be set")?; - let query = r#" - mutation CreateKnowledgeItem($input: FKnowledgeItemInput!) { - knowledge_item_create(input: $input) { - iknow_id - } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .header("User-Agent", "refact-lsp") - .json(&json!({ - "query": query, - "variables": { - "input": { - "iknow_memory": m_memory, - "located_fgroup_id": active_group_id, - "iknow_is_core": false, - "iknow_tags": vec![m_type.to_string()], - "owner_shared": false - } - } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); - } - if let Some(data) = response_json.get("data") { - if let Some(_) = data.get("knowledge_item_create") { - info!("Successfully added memory to remote server"); - return Ok(()); - } - } - Err("Failed to add memory to remote server".to_string()) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!("Failed to add memory to remote server: HTTP status {}, error: {}", status, error_text)) + tags: &[String], + filenames: &[String], + content: &str, +) -> Result { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + fs::create_dir_all(&knowledge_dir).await.map_err(|e| format!("Failed to create knowledge dir: {}", e))?; + + let filename = generate_filename(content); + let file_path = knowledge_dir.join(&filename); + + if file_path.exists() { + return Err(format!("File already exists: {}", file_path.display())); + } + + let md_content = create_markdown_content(tags, filenames, content); + fs::write(&file_path, &md_content).await.map_err(|e| format!("Failed to write knowledge file: {}", e))?; + + info!("Created knowledge entry: {}", file_path.display()); + + if let Some(vecdb) = gcx.read().await.vec_db.lock().await.as_ref() { + vecdb.vectorizer_enqueue_files(&vec![file_path.to_string_lossy().to_string()], true).await; } + + Ok(file_path) } pub async fn memories_search( gcx: Arc>, - q: &String, + query: &str, top_n: usize, ) -> Result, String> { - let client = reqwest::Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let active_group_id = gcx.read().await.active_group_id.clone() - .ok_or("active_group_id must be set")?; - let query = r#" - query KnowledgeSearch($fgroup_id: String!, $q: String!, $top_n: Int!) { - knowledge_vecdb_search(fgroup_id: $fgroup_id, q: $q, top_n: $top_n) { - iknow_id - iknow_memory - iknow_tags + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + let knowledge_prefix = knowledge_dir.to_string_lossy().to_string(); + + let has_vecdb = gcx.read().await.vec_db.lock().await.is_some(); + if has_vecdb { + let vecdb_lock = gcx.read().await.vec_db.clone(); + let vecdb_guard = vecdb_lock.lock().await; + let vecdb = vecdb_guard.as_ref().unwrap(); + let search_result = vecdb.vecdb_search(query.to_string(), top_n * 3, None).await + .map_err(|e| format!("VecDB search failed: {}", e))?; + + let mut records = Vec::new(); + for rec in search_result.results { + let path_str = rec.file_path.to_string_lossy().to_string(); + if !path_str.contains(KNOWLEDGE_FOLDER_NAME) { + continue; } - } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .header("User-Agent", "refact-lsp") - .json(&json!({ - "query": query, - "variables": { - "fgroup_id": active_group_id, - "q": q, - "top_n": top_n, + + let text = match get_file_text_from_memory_or_disk(gcx.clone(), &rec.file_path).await { + Ok(t) => t, + Err(_) => continue, + }; + + let (frontmatter, _) = MarkdownFrontmatter::parse(&text); + let lines: Vec<&str> = text.lines().collect(); + let start = (rec.start_line as usize).min(lines.len().saturating_sub(1)); + let end = (rec.end_line as usize).min(lines.len().saturating_sub(1)); + let snippet = lines[start..=end].join("\n"); + + records.push(MemoRecord { + memid: format!("{}:{}-{}", path_str, rec.start_line, rec.end_line), + tags: frontmatter.tags, + content: snippet, + file_path: Some(rec.file_path.clone()), + line_range: Some((rec.start_line, rec.end_line)), + title: frontmatter.title, + created: frontmatter.created, + }); + + if records.len() >= top_n { + break; } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); } - if let Some(data) = response_json.get("data") { - if let Some(memories_value) = data.get("knowledge_vecdb_search") { - let memories: Vec = serde_json::from_value(memories_value.clone()) - .map_err(|e| format!("Failed to parse expert: {}", e))?; - return Ok(memories); - } + + if !records.is_empty() { + return Ok(records); } - Err("Failed to get memories from remote server".to_string()) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!("Failed to get memories from remote server: HTTP status {}, error: {}", status, error_text)) } + + memories_search_fallback(gcx, query, top_n, &knowledge_prefix).await } -pub async fn memories_get_core( - gcx: Arc> +async fn memories_search_fallback( + gcx: Arc>, + query: &str, + top_n: usize, + knowledge_dir: &str, ) -> Result, String> { - let client = reqwest::Client::new(); - let api_key = gcx.read().await.cmdline.api_key.clone(); - let active_group_id = gcx.read().await.active_group_id.clone() - .ok_or("active_group_id must be set")?; - let query = r#" - query KnowledgeSearch($fgroup_id: String!) { - knowledge_get_cores(fgroup_id: $fgroup_id) { - iknow_id - iknow_memory - iknow_tags - } + let query_lower = query.to_lowercase(); + let query_words: Vec<&str> = query_lower.split_whitespace().collect(); + let mut scored_results: Vec<(usize, MemoRecord)> = Vec::new(); + + let knowledge_path = PathBuf::from(knowledge_dir); + if !knowledge_path.exists() { + return Ok(vec![]); + } + + for entry in WalkDir::new(&knowledge_path).into_iter().filter_map(|e| e.ok()) { + let path = entry.path(); + if !path.is_file() { + continue; } - "#; - let response = client - .post(&crate::constants::GRAPHQL_URL.to_string()) - .header("Authorization", format!("Bearer {}", api_key)) - .header("Content-Type", "application/json") - .header("User-Agent", "refact-lsp") - .json(&json!({ - "query": query, - "variables": { - "fgroup_id": active_group_id - } - })) - .send() - .await - .map_err(|e| format!("Failed to send GraphQL request: {}", e))?; - if response.status().is_success() { - let response_body = response - .text() - .await - .map_err(|e| format!("Failed to read response body: {}", e))?; - let response_json: Value = serde_json::from_str(&response_body) - .map_err(|e| format!("Failed to parse response JSON: {}", e))?; - if let Some(errors) = response_json.get("errors") { - let error_msg = errors.to_string(); - error!("GraphQL error: {}", error_msg); - return Err(format!("GraphQL error: {}", error_msg)); + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); + if ext != "md" && ext != "mdx" { + continue; } - if let Some(data) = response_json.get("data") { - if let Some(memories_value) = data.get("knowledge_get_cores") { - let memories: Vec = serde_json::from_value(memories_value.clone()) - .map_err(|e| format!("Failed to parse expert: {}", e))?; - return Ok(memories); - } + + let text = match get_file_text_from_memory_or_disk(gcx.clone(), &path.to_path_buf()).await { + Ok(t) => t, + Err(_) => continue, + }; + + let text_lower = text.to_lowercase(); + let score: usize = query_words.iter().filter(|w| text_lower.contains(*w)).count(); + if score == 0 { + continue; } - Err("Failed to get core memories from remote server".to_string()) - } else { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - Err(format!("Failed to get core memories from remote server: HTTP status {}, error: {}", status, error_text)) + + let (frontmatter, _) = MarkdownFrontmatter::parse(&text); + scored_results.push((score, MemoRecord { + memid: path.to_string_lossy().to_string(), + tags: frontmatter.tags, + content: text.chars().take(500).collect(), + file_path: Some(path.to_path_buf()), + line_range: None, + title: frontmatter.title, + created: frontmatter.created, + })); + } + + scored_results.sort_by(|a, b| b.0.cmp(&a.0)); + Ok(scored_results.into_iter().take(top_n).map(|(_, r)| r).collect()) +} + +pub async fn save_trajectory( + gcx: Arc>, + compressed_trajectory: &str, +) -> Result { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + let trajectories_dir = knowledge_dir.join("trajectories"); + fs::create_dir_all(&trajectories_dir).await.map_err(|e| format!("Failed to create trajectories dir: {}", e))?; + + let filename = generate_filename(compressed_trajectory); + let file_path = trajectories_dir.join(&filename); + + let tags = vec!["trajectory".to_string()]; + let md_content = create_markdown_content(&tags, &[], compressed_trajectory); + fs::write(&file_path, &md_content).await.map_err(|e| format!("Failed to write trajectory file: {}", e))?; + + info!("Saved trajectory: {}", file_path.display()); + + if let Some(vecdb) = gcx.read().await.vec_db.lock().await.as_ref() { + vecdb.vectorizer_enqueue_files(&vec![file_path.to_string_lossy().to_string()], true).await; } + + Ok(file_path) } diff --git a/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs b/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs index 45b80d46f..63d214625 100644 --- a/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs +++ b/refact-agent/engine/src/scratchpads/chat_utils_prompts.rs @@ -216,22 +216,10 @@ pub async fn system_prompt_add_extra_instructions( } if system_prompt.contains("%KNOWLEDGE_INSTRUCTIONS%") { if include_project_info { - let active_group_id = gcx.read().await.active_group_id.clone(); - if active_group_id.is_some() { - let cfg = crate::yaml_configs::customization_loader::load_customization_compiled_in(); - let mut knowledge_instructions = cfg.get("KNOWLEDGE_INSTRUCTIONS_META") - .map(|x| x.as_str().unwrap_or("").to_string()).unwrap_or("".to_string()); - if let Some(core_memories) = crate::memories::memories_get_core(gcx.clone()).await.ok() { - knowledge_instructions.push_str("\nThere are some pre-existing core memories:\n"); - for mem in core_memories { - knowledge_instructions.push_str(&format!("🗃️\n{}\n\n", mem.iknow_memory)); - } - } - system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", &knowledge_instructions); - tracing::info!("adding up extra knowledge instructions"); - } else { - system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", ""); - } + let cfg = crate::yaml_configs::customization_loader::load_customization_compiled_in(); + let knowledge_instructions = cfg.get("KNOWLEDGE_INSTRUCTIONS_META") + .map(|x| x.as_str().unwrap_or("").to_string()).unwrap_or("".to_string()); + system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", &knowledge_instructions); } else { system_prompt = system_prompt.replace("%KNOWLEDGE_INSTRUCTIONS%", ""); } diff --git a/refact-agent/engine/src/tools/tool_create_knowledge.rs b/refact-agent/engine/src/tools/tool_create_knowledge.rs index 0d14e978f..14c75f6a6 100644 --- a/refact-agent/engine/src/tools/tool_create_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_create_knowledge.rs @@ -27,17 +27,25 @@ impl Tool for ToolCreateKnowledge { }, agentic: true, experimental: false, - description: "Creates a new knowledge entry in the vector database to help with future tasks.".to_string(), + description: "Creates a new knowledge entry as a markdown file in the project's .refact_knowledge folder.".to_string(), parameters: vec![ ToolParam { - name: "knowledge_entry".to_string(), + name: "tags".to_string(), param_type: "string".to_string(), - description: "The detailed knowledge content to store. Include comprehensive information about implementation details, code patterns, architectural decisions, troubleshooting steps, or solution approaches. Document what you did, how you did it, why you made certain choices, and any important observations or lessons learned. This field should contain the rich, detailed content that future searches will retrieve.".to_string(), - } - ], - parameters_required: vec![ - "knowledge_entry".to_string(), + description: "Comma-separated tags for categorizing the knowledge entry, e.g. \"architecture, patterns, rust\"".to_string(), + }, + ToolParam { + name: "filenames".to_string(), + param_type: "string".to_string(), + description: "Comma-separated list of related file paths that this knowledge entry documents or references.".to_string(), + }, + ToolParam { + name: "content".to_string(), + param_type: "string".to_string(), + description: "The knowledge content to store. Include comprehensive information about implementation details, code patterns, architectural decisions, or solutions.".to_string(), + }, ], + parameters_required: vec!["content".to_string()], } } @@ -47,36 +55,48 @@ impl Tool for ToolCreateKnowledge { tool_call_id: &String, args: &HashMap, ) -> Result<(bool, Vec), String> { - info!("run @create-knowledge with args: {:?}", args); - let gcx = { - let ccx_locked = ccx.lock().await; - ccx_locked.global_context.clone() - }; - let knowledge_entry = match args.get("knowledge_entry") { + info!("create_knowledge {:?}", args); + + let gcx = ccx.lock().await.global_context.clone(); + + let content = match args.get("content") { Some(Value::String(s)) => s.clone(), - Some(v) => return Err(format!("argument `knowledge_entry` is not a string: {:?}", v)), - None => return Err("argument `knowledge_entry` is missing".to_string()) + Some(v) => return Err(format!("argument `content` is not a string: {:?}", v)), + None => return Err("argument `content` is missing".to_string()), + }; + + let tags: Vec = match args.get("tags") { + Some(Value::String(s)) => s.split(',') + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()) + .collect(), + Some(Value::Array(arr)) => arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect(), + _ => vec!["knowledge".to_string()], }; - crate::memories::memories_add( - gcx.clone(), - "knowledge-entry", - &knowledge_entry, - false - ).await.map_err(|e| format!("Failed to store knowledge: {e}"))?; + let tags = if tags.is_empty() { vec!["knowledge".to_string()] } else { tags }; - let mut results = vec![]; - results.push(ContextEnum::ChatMessage(ChatMessage { + let filenames: Vec = match args.get("filenames") { + Some(Value::String(s)) => s.split(',') + .map(|f| f.trim().to_string()) + .filter(|f| !f.is_empty()) + .collect(), + _ => vec![], + }; + + let file_path = crate::memories::memories_add(gcx, &tags, &filenames, &content).await?; + + Ok((false, vec![ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), - content: ChatContent::SimpleText("Knowledge entry created successfully".to_string()), + content: ChatContent::SimpleText(format!("Knowledge entry created: {}", file_path.display())), tool_calls: None, tool_call_id: tool_call_id.clone(), ..Default::default() - })); - - Ok((false, results)) + })])) } fn tool_depends_on(&self) -> Vec { - vec!["knowledge".to_string()] + vec![] } } diff --git a/refact-agent/engine/src/tools/tool_create_memory_bank.rs b/refact-agent/engine/src/tools/tool_create_memory_bank.rs index ef88365b0..c926b3c85 100644 --- a/refact-agent/engine/src/tools/tool_create_memory_bank.rs +++ b/refact-agent/engine/src/tools/tool_create_memory_bank.rs @@ -336,7 +336,7 @@ const MB_SYSTEM_PROMPT: &str = r###"• Objective: • Operational Constraint: – Do NOT call create_knowledge() until instructed."###; -const MB_EXPERT_WRAP_UP: &str = r###"Call create_knowledge() now with your complete and full analysis from the previous step if you haven't called it yet. Otherwise just type "Finished"."###; +const MB_EXPERT_WRAP_UP: &str = r###"Call create_knowledge(tags, content) now with your complete and full analysis from the previous step if you haven't called it yet. Use appropriate tags like ["architecture", "module-name", "patterns"]. Otherwise just type "Finished"."###; impl ToolCreateMemoryBank { fn build_step_prompt( @@ -497,8 +497,8 @@ impl Tool for ToolCreateMemoryBank { config_path: self.config_path.clone(), }, agentic: true, - experimental: true, - description: "Gathers information about the project structure (modules, file relations, classes, etc.) and saves this data into the memory bank.".into(), + experimental: false, + description: "Gathers information about the project structure (modules, file relations, classes, etc.) and saves this data into the knowledge base.".into(), parameters: Vec::new(), parameters_required: Vec::new(), } diff --git a/refact-agent/engine/src/tools/tool_knowledge.rs b/refact-agent/engine/src/tools/tool_knowledge.rs index 327a5622d..48e062551 100644 --- a/refact-agent/engine/src/tools/tool_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_knowledge.rs @@ -10,12 +10,10 @@ use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, Too use crate::call_validation::{ChatMessage, ChatContent, ContextEnum}; use crate::memories::memories_search; - pub struct ToolGetKnowledge { pub config_path: String, } - #[async_trait] impl Tool for ToolGetKnowledge { fn as_any(&self) -> &dyn std::any::Any { self } @@ -30,12 +28,12 @@ impl Tool for ToolGetKnowledge { }, agentic: true, experimental: false, - description: "Fetches successful trajectories to help you accomplish your task. Call each time you have a new task to increase your chances of success.".to_string(), + description: "Searches project knowledge base for relevant information. Use this to find existing documentation, patterns, decisions, and solutions.".to_string(), parameters: vec![ ToolParam { name: "search_key".to_string(), param_type: "string".to_string(), - description: "Search keys for the knowledge database. Write combined elements from all fields (tools, project components, objectives, and language/framework). This field is used for vector similarity search.".to_string(), + description: "Search query for the knowledge database. Describe what you're looking for.".to_string(), } ], parameters_required: vec!["search_key".to_string()], @@ -48,49 +46,57 @@ impl Tool for ToolGetKnowledge { tool_call_id: &String, args: &HashMap, ) -> Result<(bool, Vec), String> { - info!("run @get-knowledge {:?}", args); + info!("knowledge search {:?}", args); - let (gcx, _top_n) = { - let ccx_locked = ccx.lock().await; - (ccx_locked.global_context.clone(), ccx_locked.top_n) - }; + let gcx = ccx.lock().await.global_context.clone(); let search_key = match args.get("search_key") { Some(Value::String(s)) => s.clone(), - Some(v) => { return Err(format!("argument `search_key` is not a string: {:?}", v)) }, - None => { return Err("argument `search_key` is missing".to_string()) } + Some(v) => return Err(format!("argument `search_key` is not a string: {:?}", v)), + None => return Err("argument `search_key` is missing".to_string()), }; - let mem_top_n = 5; - let memories = memories_search(gcx.clone(), &search_key, mem_top_n).await?; - + let memories = memories_search(gcx, &search_key, 5).await?; + let mut seen_memids = HashSet::new(); let unique_memories: Vec<_> = memories.into_iter() - .filter(|m| seen_memids.insert(m.iknow_id.clone())) + .filter(|m| seen_memids.insert(m.memid.clone())) .collect(); - let memories_str = unique_memories.iter().map(|m| { - let payload: String = m.iknow_memory.clone(); - let mut combined = String::new(); - combined.push_str(&format!("🗃️{}\n", m.iknow_id)); - combined.push_str(&payload); - combined.push_str("\n\n"); - combined - }).collect::(); + let memories_str = if unique_memories.is_empty() { + "No relevant knowledge found.".to_string() + } else { + unique_memories.iter().map(|m| { + let mut result = String::new(); + if let Some(path) = &m.file_path { + result.push_str(&format!("📄 {}", path.display())); + if let Some((start, end)) = m.line_range { + result.push_str(&format!(":{}-{}", start, end)); + } + result.push('\n'); + } + if let Some(title) = &m.title { + result.push_str(&format!("📌 {}\n", title)); + } + if !m.tags.is_empty() { + result.push_str(&format!("🏷️ {}\n", m.tags.join(", "))); + } + result.push_str(&m.content); + result.push_str("\n\n"); + result + }).collect() + }; - let mut results = vec![]; - results.push(ContextEnum::ChatMessage(ChatMessage { + Ok((false, vec![ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), content: ChatContent::SimpleText(memories_str), tool_calls: None, tool_call_id: tool_call_id.clone(), ..Default::default() - })); - - Ok((false, results)) + })])) } fn tool_depends_on(&self) -> Vec { - vec!["knowledge".to_string()] + vec![] } } diff --git a/refact-agent/engine/src/tools/tools_list.rs b/refact-agent/engine/src/tools/tools_list.rs index a7d1f2794..40ffa18aa 100644 --- a/refact-agent/engine/src/tools/tools_list.rs +++ b/refact-agent/engine/src/tools/tools_list.rs @@ -39,19 +39,17 @@ fn tool_available( async fn tool_available_from_gcx( gcx: Arc>, ) -> impl Fn(&Box) -> bool { - let (ast_on, vecdb_on, allow_experimental, active_group_id) = { + let (ast_on, vecdb_on, allow_experimental) = { let gcx_locked = gcx.read().await; let vecdb_on = gcx_locked.vec_db.lock().await.is_some(); - (gcx_locked.ast_service.is_some(), vecdb_on, - gcx_locked.cmdline.experimental, gcx_locked.active_group_id.clone()) + (gcx_locked.ast_service.is_some(), vecdb_on, gcx_locked.cmdline.experimental) }; - let (is_there_a_thinking_model, allow_knowledge) = match try_load_caps_quickly_if_not_present(gcx.clone(), 0).await { - Ok(caps) => { - (caps.chat_models.get(&caps.defaults.chat_thinking_model).is_some(), active_group_id.is_some()) - }, - Err(_) => (false, false), + let is_there_a_thinking_model = match try_load_caps_quickly_if_not_present(gcx.clone(), 0).await { + Ok(caps) => caps.chat_models.get(&caps.defaults.chat_thinking_model).is_some(), + Err(_) => false, }; + let allow_knowledge = true; move |tool: &Box| { tool_available( diff --git a/refact-agent/engine/src/vecdb/mod.rs b/refact-agent/engine/src/vecdb/mod.rs index 57ce015be..e297b1fac 100644 --- a/refact-agent/engine/src/vecdb/mod.rs +++ b/refact-agent/engine/src/vecdb/mod.rs @@ -1,5 +1,6 @@ pub mod vdb_highlev; pub mod vdb_file_splitter; +pub mod vdb_markdown_splitter; pub mod vdb_structs; pub mod vdb_remote; pub mod vdb_sqlite; diff --git a/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs b/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs new file mode 100644 index 000000000..db7337908 --- /dev/null +++ b/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs @@ -0,0 +1,272 @@ +use std::path::PathBuf; +use std::sync::Arc; +use regex::Regex; +use tokio::sync::RwLock as ARwLock; + +use crate::files_in_workspace::Document; +use crate::global_context::GlobalContext; +use crate::vecdb::vdb_structs::SplitResult; +use crate::ast::chunk_utils::official_text_hashing_function; + +#[derive(Debug, Clone, Default)] +pub struct MarkdownFrontmatter { + pub title: Option, + pub tags: Vec, + pub created: Option, + pub updated: Option, +} + +impl MarkdownFrontmatter { + pub fn parse(content: &str) -> (Self, usize) { + let mut frontmatter = Self::default(); + let mut end_offset = 0; + + if content.starts_with("---") { + if let Some(end_idx) = content[3..].find("\n---") { + let yaml_content = &content[3..3 + end_idx]; + end_offset = 3 + end_idx + 4; + if content.len() > end_offset && content.as_bytes()[end_offset] == b'\n' { + end_offset += 1; + } + + for line in yaml_content.lines() { + let line = line.trim(); + if let Some(pos) = line.find(':') { + let key = line[..pos].trim(); + let value = line[pos + 1..].trim().trim_matches('"').trim_matches('\''); + match key { + "title" => frontmatter.title = Some(value.to_string()), + "created" => frontmatter.created = Some(value.to_string()), + "updated" => frontmatter.updated = Some(value.to_string()), + "tags" => { + if value.starts_with('[') && value.ends_with(']') { + frontmatter.tags = value[1..value.len() - 1] + .split(',') + .map(|s| s.trim().trim_matches('"').trim_matches('\'').to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } else if !value.is_empty() { + frontmatter.tags = vec![value.to_string()]; + } + } + _ => {} + } + } + } + } + } + (frontmatter, end_offset) + } +} + +#[derive(Debug, Clone)] +struct MarkdownSection { + heading_path: Vec, + content: String, + start_line: usize, + end_line: usize, +} + +pub struct MarkdownFileSplitter { + max_tokens: usize, + overlap_lines: usize, +} + +impl MarkdownFileSplitter { + pub fn new(max_tokens: usize) -> Self { + Self { max_tokens, overlap_lines: 3 } + } + + pub async fn split( + &self, + doc: &Document, + gcx: Arc>, + ) -> Result, String> { + let text = doc.clone().get_text_or_read_from_disk(gcx).await.map_err(|e| e.to_string())?; + let path = doc.doc_path.clone(); + let (frontmatter, content_start) = MarkdownFrontmatter::parse(&text); + let frontmatter_lines = if content_start > 0 { text[..content_start].lines().count() } else { 0 }; + let content = &text[content_start..]; + let sections = self.extract_sections(content, frontmatter_lines); + + let mut results = Vec::new(); + + if frontmatter.title.is_some() || !frontmatter.tags.is_empty() { + let frontmatter_text = self.format_frontmatter_chunk(&frontmatter); + if !frontmatter_text.is_empty() { + results.push(SplitResult { + file_path: path.clone(), + window_text: frontmatter_text.clone(), + window_text_hash: official_text_hashing_function(&frontmatter_text), + start_line: 0, + end_line: frontmatter_lines.saturating_sub(1) as u64, + symbol_path: "frontmatter".to_string(), + }); + } + } + + for section in sections { + results.extend(self.chunk_section(§ion, &path)); + } + Ok(results) + } + + fn format_frontmatter_chunk(&self, fm: &MarkdownFrontmatter) -> String { + let mut parts = Vec::new(); + if let Some(title) = &fm.title { + parts.push(format!("Title: {}", title)); + } + if !fm.tags.is_empty() { + parts.push(format!("Tags: {}", fm.tags.join(", "))); + } + parts.join("\n") + } + + fn extract_sections(&self, content: &str, line_offset: usize) -> Vec { + let heading_re = Regex::new(r"^(#{1,6})\s+(.+)$").unwrap(); + let code_fence_re = Regex::new(r"^```").unwrap(); + let mut sections = Vec::new(); + let mut current_heading_path: Vec = Vec::new(); + let mut current_content = String::new(); + let mut current_start_line = line_offset; + let mut in_code_block = false; + let lines: Vec<&str> = content.lines().collect(); + + for (idx, line) in lines.iter().enumerate() { + let absolute_line = line_offset + idx; + + if code_fence_re.is_match(line) { + in_code_block = !in_code_block; + current_content.push_str(line); + current_content.push('\n'); + continue; + } + + if in_code_block { + current_content.push_str(line); + current_content.push('\n'); + continue; + } + + if let Some(caps) = heading_re.captures(line) { + if !current_content.trim().is_empty() { + sections.push(MarkdownSection { + heading_path: current_heading_path.clone(), + content: current_content.trim().to_string(), + start_line: current_start_line, + end_line: absolute_line.saturating_sub(1), + }); + } + + let level = caps.get(1).unwrap().as_str().len(); + let heading_text = caps.get(2).unwrap().as_str().to_string(); + while current_heading_path.len() >= level { + current_heading_path.pop(); + } + current_heading_path.push(format!("{} {}", "#".repeat(level), heading_text)); + current_content = format!("{}\n", line); + current_start_line = absolute_line; + } else { + current_content.push_str(line); + current_content.push('\n'); + } + } + + if !current_content.trim().is_empty() { + sections.push(MarkdownSection { + heading_path: current_heading_path, + content: current_content.trim().to_string(), + start_line: current_start_line, + end_line: line_offset + lines.len().saturating_sub(1), + }); + } + sections + } + + fn chunk_section(&self, section: &MarkdownSection, file_path: &PathBuf) -> Vec { + let estimated_tokens = section.content.len() / 4; + + if estimated_tokens <= self.max_tokens { + return vec![SplitResult { + file_path: file_path.clone(), + window_text: section.content.clone(), + window_text_hash: official_text_hashing_function(§ion.content), + start_line: section.start_line as u64, + end_line: section.end_line as u64, + symbol_path: section.heading_path.join(" > "), + }]; + } + + self.split_large_content(§ion.content, section.start_line) + .into_iter() + .map(|(chunk_text, start, end)| SplitResult { + file_path: file_path.clone(), + window_text: chunk_text.clone(), + window_text_hash: official_text_hashing_function(&chunk_text), + start_line: start as u64, + end_line: end as u64, + symbol_path: section.heading_path.join(" > "), + }) + .collect() + } + + fn split_large_content(&self, content: &str, start_line: usize) -> Vec<(String, usize, usize)> { + let mut chunks = Vec::new(); + let lines: Vec<&str> = content.lines().collect(); + let chars_per_chunk = self.max_tokens * 4; + let mut current_chunk = String::new(); + let mut chunk_start = start_line; + let mut current_line = start_line; + + for (idx, line) in lines.iter().enumerate() { + if current_chunk.len() + line.len() + 1 > chars_per_chunk && !current_chunk.is_empty() { + chunks.push((current_chunk.trim().to_string(), chunk_start, current_line.saturating_sub(1))); + let overlap_start = idx.saturating_sub(self.overlap_lines); + current_chunk = lines[overlap_start..idx].join("\n"); + if !current_chunk.is_empty() { + current_chunk.push('\n'); + } + chunk_start = start_line + overlap_start; + } + current_chunk.push_str(line); + current_chunk.push('\n'); + current_line = start_line + idx; + } + + if !current_chunk.trim().is_empty() { + chunks.push((current_chunk.trim().to_string(), chunk_start, start_line + lines.len().saturating_sub(1))); + } + chunks + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_frontmatter_parsing() { + let content = r#"--- +title: "Test Document" +created: 2024-12-17 +tags: ["rust", "testing"] +--- + +# Hello World +"#; + let (fm, offset) = MarkdownFrontmatter::parse(content); + assert_eq!(fm.title, Some("Test Document".to_string())); + assert_eq!(fm.tags, vec!["rust", "testing"]); + assert_eq!(fm.created, Some("2024-12-17".to_string())); + assert!(offset > 0); + } + + #[test] + fn test_frontmatter_no_frontmatter() { + let content = "# Just a heading\n\nSome content"; + let (fm, offset) = MarkdownFrontmatter::parse(content); + assert!(fm.title.is_none()); + assert!(fm.tags.is_empty()); + assert_eq!(offset, 0); + } +} diff --git a/refact-agent/engine/src/vecdb/vdb_thread.rs b/refact-agent/engine/src/vecdb/vdb_thread.rs index eebfea887..54ef2a245 100644 --- a/refact-agent/engine/src/vecdb/vdb_thread.rs +++ b/refact-agent/engine/src/vecdb/vdb_thread.rs @@ -14,6 +14,7 @@ use crate::ast::file_splitter::AstBasedFileSplitter; use crate::fetch_embedding::get_embedding_with_retries; use crate::files_in_workspace::{is_path_to_enqueue_valid, Document}; use crate::global_context::GlobalContext; +use crate::vecdb::vdb_markdown_splitter::MarkdownFileSplitter; use crate::vecdb::vdb_sqlite::VecDBSqlite; use crate::vecdb::vdb_structs::{SimpleTextHashVector, SplitResult, VecDbStatus, VecdbConstants, VecdbRecord}; @@ -325,11 +326,24 @@ async fn vectorize_thread( continue; } - let file_splitter = AstBasedFileSplitter::new(constants.splitter_window_size); - let mut splits = file_splitter.vectorization_split(&doc, None, gcx.clone(), constants.embedding_model.base.n_ctx).await.unwrap_or_else(|err| { - info!("{}", err); - vec![] - }); + let is_markdown = doc.doc_path.extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .map(|e| e == "md" || e == "mdx") + .unwrap_or(false); + + let mut splits = if is_markdown { + let md_splitter = MarkdownFileSplitter::new(constants.embedding_model.base.n_ctx); + md_splitter.split(&doc, gcx.clone()).await.unwrap_or_else(|err| { + info!("{}", err); + vec![] + }) + } else { + let file_splitter = AstBasedFileSplitter::new(constants.splitter_window_size); + file_splitter.vectorization_split(&doc, None, gcx.clone(), constants.embedding_model.base.n_ctx).await.unwrap_or_else(|err| { + info!("{}", err); + vec![] + }) + }; // Adding the filename so it can also be searched if let Some(filename) = doc.doc_path.file_name().map(|f| f.to_string_lossy().to_string()) { diff --git a/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml b/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml index f5c17c660..4559e2727 100644 --- a/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml +++ b/refact-agent/engine/src/yaml_configs/customization_compiled_in.yaml @@ -55,17 +55,23 @@ SHELL_INSTRUCTIONS: | KNOWLEDGE_INSTRUCTIONS_META: | - Before any action, try to gather existing knowledge: - - Call the `knowledge()` tool to get initial information about the project and the task. - - This tool gives you access to memories, and external data, example trajectories (🗃️) to help understand and solve the task. - Always Learn and Record. Use `create_knowledge()` to: - - Important coding patterns, - - Key decisions and their reasons, - - Effective strategies, - - Insights about the project's structure and dependencies, - - When the task is finished to record useful insights. - - Take every opportunity to build and enrich your knowledge base—don’t wait for instructions. + **KNOWLEDGE MANAGEMENT** + Use the knowledge tools alongside other search tools to build and leverage project-specific knowledge: + **Reading Knowledge** - Use `knowledge(search_key)` to: + - Search for existing documentation, patterns, and decisions before starting work + - Find previous solutions to similar problems + - Understand project conventions and architectural decisions + - Retrieve saved trajectories and insights from past sessions + + **Writing Knowledge** - Use `create_knowledge(tags, content)` to save: + - Important coding patterns discovered during work + - Key architectural decisions and their rationale + - Effective strategies that worked well + - Insights about the project's structure and dependencies + - Solutions to tricky problems for future reference + + Build knowledge continuously - don't wait for explicit instructions to save useful insights. PROMPT_EXPLORATION_TOOLS: | [mode2] You are Refact Chat, a coding assistant. @@ -94,6 +100,8 @@ PROMPT_EXPLORATION_TOOLS: | %GIT_INFO% + %KNOWLEDGE_INSTRUCTIONS% + %PROJECT_TREE% @@ -331,6 +339,7 @@ subchat_tool_parameters: subchat_max_new_tokens: 128000 subchat_reasoning_effort: "high" create_memory_bank: + subchat_model_type: "light" subchat_tokens_for_rag: 120000 subchat_n_ctx: 200000 subchat_max_new_tokens: 10000 From 0f1a28b0766f219519183e9770060685e1a92718 Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 04:47:41 +1030 Subject: [PATCH 3/8] chore: Remove generated knowledge files from repo --- ...42754_refact-agent-project-architecture.md | 149 ------- ...7_043144_core-architecture-entry-points.md | 415 ------------------ ...7_044219_c-treesitter-parser-test-cases.md | 54 --- ...44254_java-treesitter-parser-test-cases.md | 54 --- ...javascript-treesitter-parser-test-cases.md | 43 -- .../2025-12-17_044355_knowledge.md | 49 --- ...418_python-treesitter-parser-test-cases.md | 36 -- ...44438_rust-treesitter-parser-test-cases.md | 41 -- ...typescript-treesitter-parser-test-cases.md | 39 -- ..._treesitter-parser-test-cases-directory.md | 53 --- ...tagentguisrcfeaturesprovidersproviderfo.md | 53 --- ...tagentguisrcfeaturesprovidersproviderfo.md | 53 --- .../2025-12-17_044542_analysis.md | 140 ------ AGENTS.md | 0 14 files changed, 1179 deletions(-) delete mode 100644 .refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md delete mode 100644 .refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md delete mode 100644 .refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md delete mode 100644 .refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md delete mode 100644 .refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md delete mode 100644 .refact_knowledge/2025-12-17_044355_knowledge.md delete mode 100644 .refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md delete mode 100644 .refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md delete mode 100644 .refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md delete mode 100644 .refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md delete mode 100644 .refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md delete mode 100644 .refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md delete mode 100644 .refact_knowledge/trajectories/2025-12-17_044542_analysis.md delete mode 100644 AGENTS.md diff --git a/.refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md b/.refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md deleted file mode 100644 index 8a48efad8..000000000 --- a/.refact_knowledge/2025-12-17_042754_refact-agent-project-architecture.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: "Refact Agent Project Architecture" -created: 2025-12-17 -tags: ["architecture", "project-structure", "rust", "lsp", "refact-agent"] ---- - -## Refact Agent Project Architecture - -### Project Overview -Refact Agent is a Rust-based LSP (Language Server Protocol) server designed to integrate with IDEs like VSCode and JetBrains. It maintains up-to-date AST (Abstract Syntax Tree) and VecDB (Vector Database) indexes for efficient code completion and project analysis. - -### Core Components - -#### 1. **refact-agent/engine** (Main Rust Application) -The heart of the project, containing: -- **src/main.rs** - Entry point for the LSP server -- **src/lsp.rs** - LSP protocol implementation -- **src/http.rs** - HTTP server for IDE communication -- **src/background_tasks.rs** - Async background task management -- **src/global_context.rs** - Shared application state - -#### 2. **Key Modules** - -**AST Module** (`src/ast/`) -- Handles Abstract Syntax Tree parsing and indexing -- Supports multiple programming languages -- Provides symbol definitions and references - -**VecDB Module** (`src/vecdb/`) -- Vector database for semantic code search -- Markdown splitting and indexing -- Efficient similarity-based code retrieval - -**AT Commands** (`src/at_commands/`) -- Special commands for IDE integration (e.g., @knowledge, @file) -- Command parsing and execution -- Context-aware completions - -**Tools** (`src/tools/`) -- External tool integrations (browsers, databases, debuggers) -- Tool authorization and execution -- Tool result processing - -**Integrations** (`src/integrations/`) -- Third-party service integrations -- API clients and handlers - -**HTTP Module** (`src/http/`) -- REST API endpoints for IDE clients -- Request/response handling -- Streaming support for long-running operations - -#### 3. **Supporting Systems** - -**Completion Cache** (`src/completion_cache.rs`) -- Caches completion results for performance -- Invalidation strategies - -**File Management** (`src/files_*.rs`) -- File filtering and blocklisting -- Workspace file discovery -- File correction and caching - -**Telemetry** (`src/telemetry/`) -- Usage tracking and analytics -- Privacy-aware data collection - -**Postprocessing** (`src/postprocessing/`) -- Result formatting and enhancement -- Output optimization - -### Project Structure - -``` -refact-agent/ -├── engine/ -│ ├── src/ -│ │ ├── agentic/ # Agentic features -│ │ ├── ast/ # AST parsing and indexing -│ │ ├── at_commands/ # IDE command handlers -│ │ ├── caps/ # Capabilities management -│ │ ├── cloud/ # Cloud integration -│ │ ├── dashboard/ # Dashboard features -│ │ ├── git/ # Git integration -│ │ ├── http/ # HTTP server -│ │ ├── integrations/ # External integrations -│ │ ├── postprocessing/ # Result processing -│ │ ├── scratchpads/ # Scratchpad management -│ │ ├── telemetry/ # Analytics -│ │ ├── tools/ # Tool integrations -│ │ ├── vecdb/ # Vector database -│ │ ├── yaml_configs/ # Configuration handling -│ │ └── main.rs, lsp.rs, http.rs, etc. -│ ├── tests/ # Python test scripts -│ ├── examples/ # Usage examples -│ └── Cargo.toml -├── gui/ # TypeScript/React frontend -└── python_binding_and_cmdline/ # Python bindings -``` - -### Key Technologies - -- **Language**: Rust (backend), TypeScript/React (frontend) -- **Protocol**: LSP (Language Server Protocol) -- **Database**: SQLite with vector extensions -- **APIs**: REST, GraphQL -- **Testing**: Python test scripts, Rust unit tests - -### Development Workflow - -1. **Building**: `cargo build` in `refact-agent/engine` -2. **Testing**: Python test scripts in `tests/` directory -3. **Running**: LSP server runs as background process -4. **Configuration**: YAML-based configuration system - -### Important Files - -- `Cargo.toml` - Rust dependencies and workspace configuration -- `known_models.json` - Supported AI models configuration -- `build.rs` - Build script for code generation -- `rustfmt.toml` - Code formatting rules - -### Integration Points - -- **IDEs**: VSCode, JetBrains (via LSP) -- **External APIs**: OpenAI, Hugging Face, custom endpoints -- **Databases**: SQLite, Vector databases -- **Version Control**: Git integration -- **Cloud Services**: Cloud-based features and authentication - -### Current Branch -- **Active**: `debug_fixes_pt_42` -- **Main branches**: `main`, `dev`, `cloud-subchats` -- **Staging**: 2 files, Modified: 23 files - -### Testing Infrastructure - -- **Unit Tests**: Rust tests in source files -- **Integration Tests**: Python scripts in `tests/` directory -- **Examples**: Executable examples in `examples/` directory -- **Test Data**: Sample data in `tests/test13_data/` - -### Performance Considerations - -- Completion caching for reduced latency -- Async background tasks for non-blocking operations -- Vector database for efficient semantic search -- Streaming responses for large outputs - diff --git a/.refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md b/.refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md deleted file mode 100644 index 8f0166d00..000000000 --- a/.refact_knowledge/2025-12-17_043144_core-architecture-entry-points.md +++ /dev/null @@ -1,415 +0,0 @@ ---- -title: "Core Architecture & Entry Points" -created: 2025-12-17 -tags: ["architecture", "core-modules", "patterns", "design", "rust", "lsp", "refact-agent", "configuration", "testing"] ---- - -## Core Architecture & Entry Points - -### Main Entry Points - -**main.rs** - Application bootstrap -- Initializes `GlobalContext` with all subsystems -- Spawns HTTP server (Axum) and LSP server (tower-lsp) -- Handles graceful shutdown and signal management -- Loads configuration from YAML files - -**lsp.rs** - Language Server Protocol implementation -- Implements tower-lsp traits for IDE communication -- Handles document synchronization -- Manages workspace symbols and definitions -- Bridges LSP requests to internal services - -**http.rs** - REST API server -- Axum-based HTTP server for IDE clients -- Endpoints for completion, chat, RAG, tools -- Streaming response support -- Request validation and error handling - -### GlobalContext - The Central Hub - -Located in `global_context.rs`, this is the "god object" that coordinates all subsystems: - -**Key Responsibilities:** -- Shared mutable state (Arc>) -- AST indexing service -- VecDB (vector database) management -- File watching and caching -- Model provider configuration -- Tool execution context -- Telemetry and analytics - -**Access Pattern:** -``` -HTTP/LSP Request → GlobalContext.read() → Service Layer → Response - → GlobalContext.write() → State Update -``` - -**Important Fields:** -- `ast_service` - AST indexing and symbol resolution -- `vecdb` - Vector database for semantic search -- `file_cache` - Completion and file caching -- `caps` - Model capabilities and providers -- `tool_executor` - Tool execution engine -- `background_tasks` - Async task management - -### Dual Protocol Architecture - -**HTTP Server (Axum)** -- Primary interface for IDE clients -- RESTful endpoints -- Streaming support for long operations -- CORS and authentication handling - -**LSP Server (tower-lsp)** -- Secondary interface for IDE integration -- Document synchronization -- Workspace symbol queries -- Hover, definition, references - -**Shared State:** -Both servers access the same `GlobalContext`, ensuring consistency across protocols. - ---- - -## Core Modules - -### AST Module (`src/ast/`) - -**Purpose:** Parse and index code structure across multiple languages - -**Key Components:** -- `AstIndexService` - Main indexing service -- Tree-sitter integration for 6+ languages (Rust, Python, JavaScript, TypeScript, Go, Java) -- Symbol definition and reference tracking -- Incremental indexing on file changes - -**Key Functions:** -- `ast_definition()` - Find symbol definition -- `ast_references()` - Find all symbol usages -- `pick_up_changes()` - Incremental indexing - -**Design Pattern:** -- Background task updates AST on file changes -- Caches results for performance -- Fallback to file content if AST unavailable - -### VecDB Module (`src/vecdb/`) - -**Purpose:** Vector database for semantic code search and RAG - -**Key Components:** -- `VecDb` - Main vector database interface -- SQLite backend with vector extensions -- Markdown splitter for code chunking -- Embedding generation and storage - -**Key Functions:** -- `search()` - Semantic similarity search -- `index()` - Add code to vector database -- `get_status()` - VecDB indexing status - -**Design Pattern:** -- Lazy initialization on first use -- Background indexing of workspace files -- Fallback to keyword search if vectors unavailable -- Configurable embedding models - -### AT Commands Module (`src/at_commands/`) - -**Purpose:** Special IDE commands for context injection and tool execution - -**Key Commands:** -- `@file` - Include file content -- `@knowledge` - Search knowledge base -- `@definition` - Find symbol definition -- `@references` - Find symbol usages -- `@web` - Web search integration -- `@tool` - Execute external tools - -**Design Pattern:** -- Command parsing and validation -- Context gathering from various sources -- Result formatting for LLM consumption -- Authorization checks for sensitive operations - -### Tools Module (`src/tools/`) - -**Purpose:** Execute external tools and integrate with external services - -**Key Tools:** -- `tool_ast_definition` - AST-based symbol lookup -- `tool_create_agents_md` - Generate project documentation -- `tool_web_search` - Web search integration -- `tool_execute_command` - Shell command execution -- Custom tool support via configuration - -**Design Pattern:** -- Tool registry with metadata -- Authorization and permission checking -- Result validation and sanitization -- Error handling and fallbacks - -### Integrations Module (`src/integrations/`) - -**Purpose:** Third-party service integrations - -**Key Integrations:** -- OpenAI API client -- Hugging Face integration -- Custom LLM endpoints -- Cloud service connections - -**Design Pattern:** -- Provider abstraction layer -- Fallback chains for redundancy -- Rate limiting and caching -- Error recovery strategies - ---- - -## Configuration System - -### YAML-Driven Configuration - -**Location:** `src/yaml_configs/` - -**Key Features:** -- Auto-generated configuration files -- Checksum validation for integrity -- Hot-reload capability -- Hierarchical configuration - -**Configuration Files:** -- `providers.yaml` - Model provider definitions -- `capabilities.yaml` - Feature capabilities -- `tools.yaml` - Tool definitions and permissions -- `integrations.yaml` - Integration settings - -**Design Pattern:** -- Configuration as code -- Validation on load -- Graceful degradation on missing configs -- Environment variable overrides - ---- - -## Performance & Caching - -### Completion Cache (`src/completion_cache.rs`) - -**Purpose:** Cache completion results for repeated queries - -**Strategy:** -- LRU cache with configurable size -- Invalidation on file changes -- Workspace-aware caching - -### File Correction Cache (`src/files_correction_cache.rs`) - -**Purpose:** Cache file correction results - -**Strategy:** -- Persistent cache with TTL -- Invalidation on file modifications - -### Background Tasks (`src/background_tasks.rs`) - -**Purpose:** Async operations without blocking main thread - -**Key Tasks:** -- AST indexing -- VecDB updates -- File watching -- Telemetry collection - -**Design Pattern:** -- Tokio-based async runtime -- Task prioritization -- Graceful shutdown handling - ---- - -## Testing Infrastructure - -### Test Organization - -**Location:** `refact-agent/engine/tests/` - -**Test Types:** -1. **Integration Tests** - Python scripts testing HTTP/LSP endpoints -2. **Unit Tests** - Rust tests in source files -3. **Examples** - Executable examples in `examples/` - -### Key Test Files - -- `test01_completion_edge_cases.py` - Completion edge cases -- `test02_completion_with_rag.py` - RAG integration -- `test03_at_commands_completion.py` - @command testing -- `test04_completion_lsp.py` - LSP protocol testing -- `test05_is_openai_compatible.py` - OpenAI API compatibility -- `test12_tools_authorize_calls.py` - Tool authorization -- `test13_vision.py` - Vision/image capabilities - -### Test Patterns - -- Python test scripts use HTTP client -- LSP tests use `lsp_connect.py` helper -- Test data in `tests/test13_data/` -- Emergency test data in `tests/emergency_frog_situation/` - ---- - -## Key Design Patterns - -### 1. Layered Fallback Pattern - -``` -Request → Cache → VecDB → AST → Model Inference - (fast) (semantic) (structural) (comprehensive) -``` - -### 2. Background Task Pattern - -``` -Main Thread (HTTP/LSP) ← Shared State → Background Threads (Indexing/Watching) -``` - -### 3. Provider Abstraction - -``` -GlobalContext → Capabilities → Provider Selection → Model Inference -``` - -### 4. Command Execution Pattern - -``` -@command → Parser → Validator → Executor → Formatter → Response -``` - -### 5. Error Recovery Pattern - -``` -Try Primary → Catch Error → Try Fallback → Log & Return Default -``` - ---- - -## Important Utilities - -### File Management (`src/files_*.rs`) - -- `files_in_workspace.rs` - Discover workspace files -- `files_blocklist.rs` - Filter blocked files -- `files_correction.rs` - Fix file paths and content -- `file_filter.rs` - Apply filtering rules - -### Privacy & Security (`src/privacy.rs`) - -- Sensitive data masking -- Privacy-aware logging -- Data sanitization - -### Telemetry (`src/telemetry/`) - -- Usage tracking -- Analytics collection -- Privacy-compliant reporting - -### Utilities - -- `tokens.rs` - Token counting and management -- `json_utils.rs` - JSON parsing helpers -- `fuzzy_search.rs` - Fuzzy matching -- `nicer_logs.rs` - Enhanced logging - ---- - -## Dependencies & Technology Stack - -### Core Dependencies - -- **tower-lsp** - LSP server implementation -- **axum** - HTTP server framework -- **tokio** - Async runtime -- **tree-sitter** - Code parsing (6+ languages) -- **sqlite-vec** - Vector database -- **serde** - Serialization -- **reqwest** - HTTP client - -### Language Support - -- Rust -- Python -- JavaScript/TypeScript -- Go -- Java -- C/C++ - -### External Services - -- OpenAI API -- Hugging Face -- Custom LLM endpoints -- Web search APIs -- Cloud services - ---- - -## Development Workflow - -### Building - -```bash -cd refact-agent/engine -cargo build --release -``` - -### Testing - -```bash -# Run all tests -cargo test --workspace - -# Run specific test -python tests/test01_completion_edge_cases.py -``` - -### Running - -```bash -cargo run --bin refact-lsp -``` - -### Configuration - -- YAML files in `src/yaml_configs/` -- Environment variables for overrides -- Hot-reload on file changes - ---- - -## Current State & Branches - -**Active Branch:** `debug_fixes_pt_42` - -**Main Branches:** -- `main` - Stable release -- `dev` - Development -- `cloud-subchats` - Cloud features -- `main-stable-2` - Previous stable - -**Staged Changes:** 2 files -**Modified Files:** 23 files - ---- - -## Key Insights - -1. **Scalability**: Dual protocol (HTTP + LSP) allows flexible IDE integration -2. **Performance**: Multi-layer caching and background indexing minimize latency -3. **Extensibility**: YAML-driven configuration enables easy feature addition -4. **Reliability**: Fallback chains and error recovery ensure graceful degradation -5. **Maintainability**: Clear separation of concerns across modules -6. **Testing**: Comprehensive integration tests validate functionality - diff --git a/.refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md deleted file mode 100644 index bf65ad05b..000000000 --- a/.refact_knowledge/2025-12-17_044219_c-treesitter-parser-test-cases.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: "C++ Tree-sitter Parser Test Cases" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "cpp", "refact-agent"] ---- - -### C++ Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains test fixtures (sample C++ source files and their expected outputs) specifically for validating the C++ Tree-sitter parser implementation within the refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world C++ code for features like "go to definition" (`ast_definition`), "find references" (`ast_references`), and code analysis in the LSP server. The tests build upon the core AST module's incremental indexing and multi-language support by providing language-specific validation data. - -**• Files:** -``` -cpp/ -├── circle.cpp # Sample C++ source: likely tests class/struct definitions, methods -├── circle.cpp.decl_json # Expected JSON output: extracted declarations/symbol table -├── circle.cpp.skeleton # Expected skeletonized version: structure-only (no bodies/implementation) -├── main.cpp # Sample C++ source: entry point, function calls, includes -└── main.cpp.json # Expected JSON output: full AST parse or symbol info -``` -- **Organization pattern**: Each test case pairs a `.cpp` source file with companion `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This mirrors test setups in sibling directories (`java/`, `python/`, etc.), enabling consistent cross-language validation. -- **Naming convention**: `filename.lang[.variant].{json|decl_json|skeleton}` – clear, machine-readable, focused on parser outputs. -- Notable: Directory reported as "empty" in file read, but structure confirms 5 fixture files present for targeted C++ testing. - -**• Architecture:** -- **Role in AST pipeline**: Part of `src/ast/treesitter/parsers/tests/cases/` hierarchy, consumed by `tests/cpp.rs` (test runner module). Tests invoke Tree-sitter's C++ grammar (`parsers/cpp.rs`) to parse files, then validate against golden `.json`/`.skeleton` outputs. -- **Design patterns**: - - **Golden file testing**: Compare runtime parser output vs. pre-approved fixtures for regression-proofing. - - **Modular language isolation**: Per-language subdirs allow independent grammar evolution without affecting others (e.g., Rust/Python parsers unchanged). - - **Multi-output validation**: Tests three concerns simultaneously – raw AST (`json`), symbols (`decl_json`), structure (`skeleton`) – covering the full AST-to-analysis flow. -- **Data flow**: `file_ast_markup.rs` or `skeletonizer.rs` processes `.cpp` → generates AST/symbols → serializes to JSON → `tests/cpp.rs` asserts equality. -- **Fits layered architecture**: Bottom layer (Tree-sitter parsing) → tested here → feeds `ast_db.rs`/`ast_indexer_thread.rs` for background indexing → used by `at_ast_definition.rs`/`at_ast_reference.rs`. -- **Error handling**: Implicit via test failures; likely uses `anyhow` or custom `custom_error.rs` for parse errors. -- **Extension points**: Easy to add new C++ edge cases (templates, lambdas, STL) without code changes. - -**• Key Symbols (inferred from test consumers):** -- From `ast_instance_structs.rs`/`structs.rs`: `AstInstance`, `SymbolDecl`, `SkeletonNode` – parsed/validated here. -- Parser entry: `parsers/cpp.rs::parse_cpp()` or similar – Tree-sitter query capture for C++ nodes. -- Test harness: `tests/cpp.rs` likely exports `test_cpp_cases()` calling `language_id.rs::Cpp`, `file_ast_markup.rs::markup_file()`. -- Cross-references: Relies on `parse_common.rs`, `utils.rs`; outputs feed `ast_structs.rs::Node`. - -**• Integration:** -- **Used by**: `src/ast/treesitter/parsers/tests/cpp.rs` (direct test runner); indirectly powers `at_ast_definition.rs`, `tool_ast_definition.rs`, `ast_indexer_thread.rs` for live IDE queries. -- **Uses from others**: Tree-sitter grammars (`parsers/cpp.rs`), shared utils (`treesitter/utils.rs`, `chunk_utils.rs`), `language_id.rs` for C++ detection. -- **Relationships**: - | Depends On | Used By | Communication | - |---------------------|-----------------------------|---------------| - | `parsers/cpp.rs` | `tests/cpp.rs` | File paths, parse results | - | `skeletonizer.rs` | `ast_db.rs` | Skeleton strings | - | `language_id.rs` | `ast_parse_anything.rs` | Language enum (Cpp) | -- **Comes after**: General `alt_testsuite/` (annotated complex cases like `cpp_goat_library.cpp`); more focused than multi-lang `tests.ts`. -- **Comparison to existing knowledge**: Builds upon AST module's "Tree-sitter integration for 6+ languages" (core arch doc) by providing C++-specific fixtures. Unlike broader `ast_indexer_thread.rs` (runtime indexing), this is pure parser validation. Introduces language-specific golden testing pattern seen across `js/`, `rust/`, etc., enabling "pick_up_changes()" incremental updates with confidence. - -This test suite ensures C++ parsing reliability in the agent's multi-language AST system, critical for production IDE features like symbol navigation. diff --git a/.refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md deleted file mode 100644 index 749523623..000000000 --- a/.refact_knowledge/2025-12-17_044254_java-treesitter-parser-test-cases.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: "Java Tree-sitter Parser Test Cases" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "java", "refact-agent"] ---- - -### Java Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains test fixtures (sample Java source files and their expected outputs) specifically for validating the Java Tree-sitter parser implementation within refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world Java code for features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), and code analysis in the LSP server. The tests build upon the core AST module's incremental indexing (`ast_indexer_thread.rs`) and multi-language support by providing Java-specific validation data, mirroring the pattern seen in sibling directories like `cpp/`, `python/`, etc. - -**• Files:** -``` -java/ -├── main.java # Sample Java source: likely entry point with class definitions, methods, imports -├── main.java.json # Expected JSON output: full AST parse or symbol info -├── person.java # Sample Java source: tests class/struct definitions, fields, constructors -├── person.java.decl_json # Expected JSON output: extracted declarations/symbol table -└── person.java.skeleton # Expected skeletonized version: structure-only (no bodies/implementation) -``` -- **Organization pattern**: Each test case pairs a `.java` source file with companion `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This enables consistent cross-language validation across `cases/` subdirectories (`cpp/`, `js/`, `kotlin/`, `python/`, `rust/`, `ts/`). -- **Naming convention**: `filename.lang[.variant].{json|decl_json|skeleton}` – clear, machine-readable, focused on parser outputs (e.g., `decl_json` for symbol tables, `skeleton` for structural stripping). -- **Notable details**: Simple, focused examples (`main` + `person`) cover core Java constructs like classes, methods, and fields. Directory reported as "empty" in file read, but structure confirms 5 fixture files present for targeted Java testing. - -**• Architecture:** -- **Module role**: Part of the AST subsystem (`src/ast/treesitter/parsers/tests/`), which uses golden-file testing (source + expected outputs) to validate Tree-sitter parsers. Follows a layered pattern: raw Tree-sitter grammars (`parsers/java.rs`) → parse/symbol extraction (`file_ast_markup.rs`, `ast_instance_structs.rs`) → skeletonization (`skeletonizer.rs`) → indexing (`ast_db.rs`). -- **Design patterns**: Golden testing (compare actual vs. expected outputs); language-specific isolation for multi-lang support; incremental parsing validation to support live IDE updates via `ast_parse_anything.rs`. -- **Relationships**: Sibling to `cpp/`, `kotlin/` (OO languages with similar class/method structures). Fits into refact-agent's layered architecture: AST layer feeds tools/AT commands → HTTP/LSP handlers → agentic features. -- **Comes after**: Broader `alt_testsuite/` (complex annotated cases); more focused than multi-lang `tests.rs`. -- **Comparison to existing knowledge**: Directly analogous to C++ test cases (from knowledge base), which use identical structure (`circle.cpp`/`main.cpp` → `person.java`/`main.java`). Builds upon AST module's "Tree-sitter integration for 6+ languages" by adding Java-specific fixtures. Unlike runtime indexing (`ast_indexer_thread.rs`), this is pure parser validation via `tests/java.rs`. Introduces consistent golden testing pattern across OO languages, enabling reliable "pick_up_changes()" incremental updates. - -**• Key Symbols:** -- No runtime symbols (pure data files), but validates parser outputs feeding: - | Symbol/Path | Purpose | - |--------------------------|----------------------------------| - | `ast_structs.rs::Node` | Stores parsed AST nodes | - | `language_id.rs::Java` | Language enum for detection | - | `skeletonizer.rs` | Generates `.skeleton` files | - | `parsers/java.rs` | Tree-sitter grammar/query defs | - -**• Integration:** -- **Used by**: `src/ast/treesitter/parsers/tests/java.rs` (direct test runner loads these files, parses, compares JSON/skeletons); indirectly powers `at_ast_definition.rs`, `tool_ast_definition.rs`, `ast_indexer_thread.rs` for live IDE queries (e.g., go-to-definition in Java projects). -- **Uses from others**: Tree-sitter grammars (`parsers/java.rs`), shared utils (`treesitter/utils.rs`, `chunk_utils.rs`, `parse_common.rs`), `language_id.rs` for Java detection. -- **Relationships**: - | Depends On | Used By | Communication | - |-----------------------|-------------------------------|------------------------| - | `parsers/java.rs` | `tests/java.rs` | File paths, parse results | - | `skeletonizer.rs` | `ast_db.rs` | Skeleton strings | - | `language_id.rs` | `ast_parse_anything.rs` | Language enum (Java) | - | `file_ast_markup.rs` | `at_ast_reference.rs` | Symbol tables (decl_json) | -- **Data flow**: Fixtures → `tests/java.rs` (parse → serialize → assert_eq!) → confidence in `ast_db.rs` insertion → runtime queries via LSP/HTTP (`v1/ast.rs`). -- **Cross-cutting**: Error handling via parse failures in tests; supports multi-language AST index used by VecDB (`vecdb/`) and agent tools (`tools/tool_ast_definition.rs`). - -This test suite ensures Java parsing reliability in the agent's multi-language AST system, critical for production IDE features like symbol navigation in Java/Kotlin projects. diff --git a/.refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md deleted file mode 100644 index 2536d40ff..000000000 --- a/.refact_knowledge/2025-12-17_044322_javascript-treesitter-parser-test-cases.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: "JavaScript Tree-sitter Parser Test Cases" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "javascript", "js", "refact-agent"] ---- - -### JavaScript Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains test fixtures (sample JavaScript source files and their expected outputs) specifically for validating the JavaScript Tree-sitter parser implementation within refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world JavaScript code for features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), code analysis, and agentic tools in the LSP server. The tests build upon the core AST module's incremental indexing (`ast_indexer_thread.rs`) and multi-language support by providing JavaScript-specific validation data, following the exact pattern of sibling directories like `cpp/`, `java/`, `python/`, etc. - -**• Files:** -``` -js/ -├── car.js # Sample JS source: likely tests object literals, functions, prototypes, or ES6+ features -├── car.js.decl_json # Expected JSON output: extracted declarations/symbol table (functions, vars, exports) -├── car.js.skeleton # Expected skeletonized version: structure-only (no function bodies/implementation details) -├── main.js # Sample JS source: entry point with modules, imports/exports, async functions, classes -└── main.js.json # Expected JSON output: full AST parse or complete symbol info -``` -- **Organization pattern**: Identical to other language test dirs—each `.js` source pairs with `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This enables consistent cross-language golden testing across `cases/` subdirectories (`cpp/`, `java/`, `kotlin/`, `python/`, `rust/`, `ts/`), loaded by `tests/js.rs`. - -**• Architecture:** -- **Design patterns**: Golden testing (compare parser output vs. expected files); language-isolated validation for scalable multi-lang support (7+ languages via `language_id.rs`); supports incremental parsing for live IDE reloads (`ast_parse_anything.rs`, `ast_indexer_thread.rs`). Fits refact-agent's layered architecture: AST parsers → `ast_structs.rs` nodes → tools/AT commands (`at_ast_*`) → HTTP/LSP handlers → agentic workflows. -- **Module relationships**: Consumed by `treesitter/parsers/tests/js.rs` for unit tests; feeds `parsers/js.rs` (language-specific queries/grammars); upstream from `file_ast_markup.rs` and `skeletonizer.rs`. Sibling to `ts/` (TypeScript, sharing JS grammar base). -- **Comparison to existing knowledge**: Directly analogous to documented Java/C++ cases (`person.java`/`circle.cpp` → `person.js`/`car.js` naming for "entity modeling"). Unlike broader `alt_testsuite/` (edge-case annotated files like `py_torture*.py`), this focuses on canonical parser validation. Builds upon "Tree-sitter integration for 6+ languages" (from project architecture knowledge) by extending to dynamic/scripting langs like JS. Introduces JS-specific challenges (hoisting, closures, dynamic imports) vs. static OO langs, enabling reliable incremental updates via `pick_up_changes()`. - -**• Key Symbols:** -- No runtime symbols (pure data fixtures), but validates outputs feeding core AST pipeline: - | Symbol/Path | Purpose | - |------------------------------|----------------------------------------------| - | `ast_structs.rs::Node` | Parsed AST nodes from JS source | - | `language_id.rs::JavaScript` | Language enum for `.js` detection | - | `skeletonizer.rs` | Generates `.skeleton` files (structure only) | - | `parsers/js.rs` | JS-specific Tree-sitter grammar/capture queries | - | `tests/js.rs` | Test runner loading these fixtures | - | `ast_db.rs` | Stores validated JS symbols in workspace DB | - -**• Integration:** -- **Used by**: `treesitter/parsers/tests/js.rs` (direct test loader); indirectly powers `@ast-definition`/`@ast-reference` AT commands, code completion (`scratchpads/code_completion_*`), and RAG via `vecdb/*`. -- **Uses**: Tree-sitter JS grammar (external crate); `parse_common.rs` for shared parsing logic. -- **Communication**: Fixtures → test assertions → parser validation → runtime AST indexing (`ast_indexer_thread.rs`). -- **Dependencies**: Part of AST layer; no external deps beyond Tree-sitter. Extension point: Add new fixtures for JS edge cases (e.g., React JSX via queries in `parsers/js.rs`). In broader refact-agent flow: Workspace files → these parsers → `tools/tool_ast_*` → agent chat/tools. diff --git a/.refact_knowledge/2025-12-17_044355_knowledge.md b/.refact_knowledge/2025-12-17_044355_knowledge.md deleted file mode 100644 index 1f44fce09..000000000 --- a/.refact_knowledge/2025-12-17_044355_knowledge.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: "---" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "kotlin", "refact-agent"] ---- - ---- -title: \"Kotlin Tree-sitter Parser Test Cases\" -created: 2025-12-17 -tags: [\"architecture\", \"ast\", \"treesitter\", \"parsers\", \"tests\", \"kotlin\", \"refact-agent\"] ---- - -### Kotlin Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains test fixtures (sample Kotlin source files and their expected outputs) for validating the Kotlin Tree-sitter parser within refact-agent's AST system. It ensures accurate syntax tree generation, symbol extraction (declarations/definitions), and skeletonization (structure-only versions without implementation details). These tests support Kotlin-specific code intelligence features like \"go to definition\" (`at_ast_definition.rs`), \"find references\" (`at_ast_reference.rs`), and incremental indexing (`ast_parse_anything.rs`, `ast_indexer_thread.rs`) in the LSP server. As part of the multi-language AST layer, it enables reliable parsing for JVM ecosystems, feeding into agentic tools, HTTP/LSP handlers, and VecDB indexing. - -**• Files:** -``` -kotlin/ -├── main.kt # Sample Kotlin source: entry point testing functions, classes, companions, or top-level declarations -├── main.kt.json # Expected JSON: full AST parse or symbol info (e.g., function signatures, imports) -├── person.kt # Sample Kotlin source: class/data class definitions, properties, constructors, extensions -├── person.kt.decl_json # Expected JSON: extracted declarations/symbol table (fields, methods, overrides) -├── person.kt.json # Expected JSON: complete parse/symbol info for person.kt (complements decl_json) -└── person.kt.skeleton # Expected skeleton: structure-only (signatures, hierarchy; no bodies) -``` -- **Organization pattern**: Follows the consistent golden testing format across `cases/` siblings (`cpp/`, `java/`, `js/`, `python/`, `rust/`, `ts/`): each `.kt` source pairs with `.json` (parse/symbols), `.decl_json` (declarations), and `.skeleton` files. Loaded by `tests/kotlin.rs` for automated validation. - -**• Architecture:** -- **Design patterns**: Golden file testing (actual vs. expected outputs for parser stability); language isolation in `parsers/kotlin.rs`; incremental validation for live IDE updates. Fits refact-agent's layered architecture: AST layer (`treesitter/`) → AT commands/tools → HTTP/LSP handlers (`http/routers/v1/ast.rs`) → agentic features. -- **Relationships**: Sibling to `java/` (similar OO/JVM patterns like classes/methods); uses `language_id.rs::Kotlin`; outputs feed `ast_structs.rs::Node`, `skeletonizer.rs`. Unlike broader `alt_testsuite/` (annotated edge cases), this focuses on core parser accuracy. -- **Comparison to existing knowledge**: Directly analogous to Java/JS test cases—same structure (`person.java`/`car.js` → `person.kt`), building on \"Tree-sitter integration for 6+ languages\" by adding Kotlin-specific fixtures. Unlike runtime indexing, pure validation via `tests/kotlin.rs`. Introduces JVM language consistency, enabling cross-lang symbol resolution. - -**• Key Symbols:** -| Symbol/Path | Purpose | -|------------------------------|----------------------------------------------| -| `ast_structs.rs::Node` | Parsed AST node storage | -| `language_id.rs::Kotlin` | Language detection/enum | -| `parsers/kotlin.rs` | Kotlin-specific Tree-sitter grammar/queries | -| `skeletonizer.rs` | Generates `.skeleton` (structure extraction) | -| `tests/kotlin.rs` | Test runner loading these fixtures | -| `file_ast_markup.rs` | Markup/symbol extraction from parse trees | - -**• Integration:** -- **Uses**: Tree-sitter external crate; core AST utils (`parse_common.rs`, `ast_instance_structs.rs`). -- **Used by**: `at_ast_definition.rs`, `at_ast_reference.rs` (navigation tools); `ast_indexer_thread.rs` (background indexing); LSP handlers (`lsp_like_handlers.rs`); VecDB (`vecdb/` for code search). -- **Communication**: Fixtures → `tests/kotlin.rs` → parser validation → runtime AST feeds (`ast_db.rs`). Supports multi-lang workspace analysis via `ast_parse_anything.rs`. -- **Extension points**: New fixtures easily added for Kotlin features (coroutines, sealed classes); pattern extensible to other langs. Error handling via `custom_error.rs` patterns in parser layer. diff --git a/.refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md deleted file mode 100644 index f25f80931..000000000 --- a/.refact_knowledge/2025-12-17_044418_python-treesitter-parser-test-cases.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "Python Tree-sitter Parser Test Cases" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "python", "refact-agent"] ---- - -### Python Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains test fixtures (sample Python source files and their expected outputs) specifically for validating the Python Tree-sitter parser implementation within refact-agent's AST system. It enables automated testing of syntax tree generation, symbol extraction (declarations, definitions), and skeletonization (stripping implementation details while preserving structure). These tests ensure the parser accurately handles real-world Python code for features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), code analysis, and agentic tools in the LSP server. The tests build upon the core AST module's incremental indexing (`ast_indexer_thread.rs`) and multi-language support by providing Python-specific validation data, following the exact pattern of sibling directories like `cpp/`, `java/`, `js/`, `kotlin/`, `rust/`, and `ts/`. - -**• Files:** -``` -python/ -├── calculator.py # Sample Python source: likely tests arithmetic expressions, functions, classes, or decorators -├── calculator.py.decl_json # Expected JSON output: extracted declarations/symbol table (functions, classes, globals) -├── calculator.py.skeleton # Expected skeletonized version: structure-only (no function bodies/implementation details) -├── main.py # Sample Python source: entry point with imports, modules, comprehensions, or async code -└── main.py.json # Expected JSON output: full AST parse or complete symbol info -``` -- **Organization pattern**: Identical to other language test dirs—each `.py` source pairs with `.json` (parse/symbol results) and `.skeleton` (structure-only) files. This enables consistent cross-language golden testing across `cases/` subdirectories, loaded by `tests/python.rs`. - -**• Architecture:** -- **Golden file testing pattern**: Follows a uniform "source + expected output" strategy across all languages, ensuring parser reliability via snapshot-style tests. The `parsers/python.rs` module uses these to validate Tree-sitter parsing against pre-computed `.json` (AST/symbols) and `.skeleton` (minified structure) baselines. -- **Fits into layered AST architecture**: Part of the `ast/treesitter/parsers/tests/` testing layer, which validates the parsing layer (`parsers/*.rs`) before feeding into higher layers like indexing (`ast_indexer_thread.rs`), @-commands (`at_ast_*`), and tools (`tool_ast_definition.rs`). -- **Design patterns**: Test-Driven Development (TDD) with golden files; language-agnostic test harness in `tests/*.rs` modules that discovers and runs cases dynamically. - -**• Key Symbols:** -- No runtime symbols (pure test data), but tests validate parser outputs for Python-specific Tree-sitter nodes like `function_definition`, `class_definition`, `arguments`, `parameters`, `async_function_definition`. -- Loaded by test functions in `tests/python.rs`, which likely call `parsers/python.rs` entrypoints like `parse_file()` or `extract_declarations()` and assert against `.json`/`.skeleton`. - -**• Integration:** -- **Used by**: `tests/python.rs` (test runner); indirectly supports `at_ast_definition.rs`, `at_ast_reference.rs`, `tool_ast_definition.rs`, and `ast_db.rs` by ensuring parser correctness. -- **Uses**: Tree-sitter Python grammar (via `parsers/python.rs`); core AST structs from `treesitter/structs.rs` and `ast_structs.rs`. -- **Relationships**: Mirrors sibling dirs (e.g., `java/`, `js/`), enabling unified test suite execution. Feeds into LSP handlers (`http/routers/v1/ast.rs`) for features like `/ast` endpoints. Part of broader AST pipeline: raw parse → symbol extraction → indexing → agent tools. -- **This builds upon**: Multi-language parser validation pattern seen in documented `java/` and `js/` cases, extending it to Python for comprehensive language coverage in refact-agent's AST system. Unlike generic tests, these focus on Python idioms (e.g., decorators, type hints, f-strings) critical for accurate code intelligence. diff --git a/.refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md deleted file mode 100644 index 1d4452074..000000000 --- a/.refact_knowledge/2025-12-17_044438_rust-treesitter-parser-test-cases.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: "Rust Tree-sitter Parser Test Cases" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "rust", "refact-agent"] ---- - -### Rust Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains Rust-specific test fixtures (source files paired with expected outputs) for validating the Tree-sitter parser implementation in refact-agent's AST system. It tests accurate parsing of Rust syntax into ASTs, extraction of symbols (declarations/definitions), and skeletonization (structural abstraction without implementation details). These golden tests ensure the parser supports Rust code analysis features like "go to definition" (`at_ast_definition.rs`), "find references" (`at_ast_reference.rs`), incremental indexing (`ast_indexer_thread.rs`), and agentic tooling. As part of a consistent cross-language testing strategy, it mirrors sibling directories (`cpp/`, `java/`, `js/`, `kotlin/`, `python/`, `ts/`), enabling uniform validation loaded by `tests/rust.rs`. - -**• Files:** -``` -rust/ -├── main.rs # Sample Rust entry point: tests modules, functions, traits, impls, enums -├── main.rs.json # Expected full AST parse or symbol table JSON (complete node structure/decls) -├── point.rs # Sample Rust module: likely tests structs, methods, generics, lifetimes -├── point.rs.decl_json # Expected declarations JSON: extracted symbols (structs, fns, types, visibilities) -└── point.rs.skeleton # Expected skeletonized source: structure-only (signatures without bodies) -``` -- **Naming pattern**: `*.rs` sources → `*.rs.json` (full parse/symbols), `*.rs.decl_json` (decl-only), `*.skeleton` (abstraction). Pairs enable precise diff-based assertions in `tests/rust.rs`. -- **Organization**: Matches all `cases/*/` dirs—minimal, focused samples covering core language features (e.g., ownership, traits, async) without complexity. - -**• Architecture:** -- **Single Responsibility**: Pure data-driven testing—no logic, just inputs/outputs for parser black-box validation. -- **Pattern**: Golden file testing (source → expected AST/symbols/skeleton). Builds upon core AST pipeline: `parsers/rust.rs` → `file_ast_markup.rs`/`skeletonizer.rs` → `ast_instance_structs.rs`. -- **Relationships**: - - **Used by**: `treesitter/parsers/tests/rust.rs` (test runner loads/parses/compares). - - **Uses**: None (static data); integrates with `language_id.rs`, `ast_structs.rs`. - - **Cross-lang consistency**: Identical to JS (`car.js/main.js`), Python (`calculator.py`), etc.—"This follows the exact pattern from `js/`, `java/`, etc., introducing Rust-specific handling (e.g., lifetimes, impls) unlike simpler langs." -- **Layered fit**: Data layer → AST module (`src/ast/treesitter/`) → AT commands/tools → LSP/HTTP handlers. - -**• Key Symbols:** -- No code; fixtures validate parser-extracted symbols like `struct Point`, `fn main()`, `impl Point`, visibilities (`pub`), lifetimes (`'a`). -- Tests `ast_structs.rs` types: `AstInstance`, declaration nodes, spans. - -**• Integration:** -- **Data Flow**: `rust.rs` test reads files → invokes `parsers/rust.rs`/`skeletonizer.rs` → asserts against `.json`/`.skeleton`. -- **Dependencies**: Tree-sitter Rust grammar; shared utils (`utils.rs`, `structs.rs`). -- **Extension**: New fixtures added for grammar updates; supports multi-lang AST DB (`ast_db.rs`). -- **Unlike others**: Rust tests emphasize ownership/traits vs. JS prototypes or Python dynamics, but shares abstraction boundaries for uniform `at_*` tools. This completes the 7-lang suite (`cpp/java/js/kotlin/python/rust/ts`), enabling comprehensive parser reliability in the LSP agent. diff --git a/.refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md b/.refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md deleted file mode 100644 index 453f80d83..000000000 --- a/.refact_knowledge/2025-12-17_044512_typescript-treesitter-parser-test-cases.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "TypeScript Tree-sitter Parser Test Cases" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "typescript", "ts", "refact-agent"] ---- - -### TypeScript Tree-sitter Parser Test Cases - -**• Purpose:** -This directory contains test fixtures (sample TypeScript source files paired with their expected parser outputs) for validating the TypeScript Tree-sitter parser implementation in refact-agent's AST processing pipeline. It ensures accurate syntax tree generation, symbol extraction (declarations/definitions), and skeletonization (structure-preserving code stripping) specifically for TypeScript/JavaScript variants. These tests support critical agent features like `@ast-definition` (jump-to-definition), `@ast-reference` (find usages), code navigation via `@tree`, and AST-driven RAG/context retrieval. As part of the multi-language test suite under `cases/`, it follows an identical golden-file pattern to siblings (`cpp/`, `java/`, `js/`, `kotlin/`, `python/`, `rust/`), enabling consistent, automated validation loaded by `tests/ts.rs`. This builds upon the core AST indexer (`ast_indexer_thread.rs`) and parser registry (`parsers/parsers.rs` → `ts.rs`), providing TypeScript-specific edge cases like interfaces, generics, decorators, and type-only imports/exports. - -**• Files:** -``` -ts/ -├── main.ts # Sample entry-point source: tests modules, imports/exports, classes, interfaces, async functions -├── main.ts.json # Expected full AST parse or complete symbol table (functions, types, exports in JSON) -├── person.ts # Sample source: likely tests type definitions, interfaces, generics, or OOP patterns -├── person.ts.decl_json # Expected declarations/symbol table (vars, types, methods with locations/scopes) -└── person.ts.skeleton # Expected skeletonized output: structural code only (no impl details, comments, literals) -``` -- **Organization pattern**: Matches all `cases/*/` dirs precisely—source files (`.ts`) pair with `.json`/`.decl_json` (parse/symbol golden results) and `.skeleton` (structure-only). Minimalist (2 sources), focused on core TS constructs vs. broader JS coverage in sibling `js/`. No subdirs; flat for simple test loading. - -**• Architecture:** -- **Role in layered architecture**: Test/data layer for the AST module (`src/ast/treesitter/parsers/`), validating the parser abstraction boundary in `parsers.ts.rs`. Uses Tree-sitter's incremental parsing via `tree-sitter` crate, integrated with `ast_instance_structs.rs` for typed nodes and `file_ast_markup.rs` for markup/symbol extraction. -- **Design patterns**: Golden-file testing (expected outputs as data-driven tests); cross-language uniformity (shared loader in `tests/*.rs`); separation of parse/symbol/skeleton concerns. -- **Data flow**: Fixtures → `ts.rs` test module → parser (`ts.rs`) → AST build → JSON serialization/symbol extraction → skeletonizer (`skeletonizer.rs`) → assert equality. -- **Relationships**: - - **Used by**: `tests/ts.rs` (test runner); indirectly `at_ast_definition.rs`, `at_ast_reference.rs`, `tool_ast_definition.rs` (via indexed AST DB). - - **Uses**: Parser registry (`parsers/parsers.rs` → language_id.rs` for TS detection); AST structs (`structs.rs`). - - **Integration**: Feeds `ast_db.rs` / `ast_indexer_thread.rs` for workspace indexing; powers HTTP/LSP endpoints (`http/routers/v1/ast.rs`). - -**• Key Symbols (inferred from parser/tests context):** -- **Parsers**: `get_typescript_parser()` in `parsers/ts.rs` (loads Tree-sitter TS grammar). -- **Test fns**: In `tests/ts.rs` – `test_parse()`, `test_declarations()`, `test_skeletonize()` (load fixtures, assert outputs). -- **AST Types**: `AstInstance`, `SymbolDecl` (from `ast_instance_structs.rs` / `structs.rs`). -- **Utilities**: `file_ast_markup.rs` (markup gen), `skeletonizer.rs` (structure extraction). - -**• Integration:** -Fits Refact Agent's AST-centric architecture by providing TS-specific validation for the multi-lang parser system. Unlike pure JS tests (`js/`), emphasizes type system handling (e.g., `person.ts` likely tests interfaces/types). Builds upon general AST patterns from `parse_common.rs` but specializes via `language_id.rs` (TS vs. JS detection). Dependencies flow upward to agent tools (`tools/tool_ast_*.rs`), `@at` commands (`at_ast_*.rs`), and indexing (`ast_parse_anything.rs`). Ensures reliability for IDE features (LSP code lens, go-to-def) and agentic reasoning over TS projects. No runtime deps—pure test data. diff --git a/.refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md b/.refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md deleted file mode 100644 index 55a01a47b..000000000 --- a/.refact_knowledge/2025-12-17_044542_treesitter-parser-test-cases-directory.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: "Tree-sitter Parser Test Cases Directory Analysis" -created: 2025-12-17 -tags: ["architecture", "ast", "treesitter", "parsers", "tests", "test-cases", "refact-agent"] ---- - -### Tree-sitter Parser Test Cases Directory Analysis - -**• Purpose:** -This directory serves as the test data repository for the Tree-sitter parser implementations in the refact-agent's AST (Abstract Syntax Tree) subsystem. It contains language-specific sample source files, their corresponding parsed JSON outputs, and skeletonized representations. The primary goal is to validate parser accuracy, ensure consistent AST generation across languages, and test features like symbol extraction (declarations) and code skeletonization. These tests verify that Tree-sitter grammars correctly handle real-world code constructs for languages supported by the agent (C++, Java, JavaScript, Kotlin, Python, Rust, TypeScript), enabling reliable code analysis, completions, definitions/references (@ast_definition, @ast_reference), and indexing in the broader AST pipeline. - -**• Files:** -Organized by language in subdirectories (`cpp/`, `java/`, `js/`, `kotlin/`, `python/`, `rust/`, `ts/`), each containing minimal but representative code samples and their parser outputs: -- **Source files** (e.g., `main.cpp`, `circle.cpp`, `calculator.py`, `main.rs`): Simple, self-contained programs demonstrating key language features (classes, functions, imports, OOP constructs). -- **JSON dumps** (e.g., `main.cpp.json`, `person.java.decl_json`): Full AST serialization from Tree-sitter queries, capturing node hierarchies, spans, and metadata. -- **Declaration JSONs** (e.g., `circle.cpp.decl_json`, `person.kt.decl_json`): Extracted symbol tables focusing on definitions (functions, classes, variables). -- **Skeleton files** (e.g., `circle.cpp.skeleton`, `car.js.skeleton`): Simplified code representations stripping bodies/details, used for RAG/indexing previews or diff analysis. - -| Language | Key Files | Purpose | -|----------|-----------|---------| -| C++ (`cpp/`) | `main.cpp`, `circle.cpp`, `.json`/`.decl_json`/`.skeleton` | Tests class/method parsing, includes. | -| Java (`java/`) | `main.java`, `person.java` + outputs | OOP inheritance, constructors. | -| JS (`js/`) | `main.js`, `car.js` + outputs | Prototypes, closures, modules. | -| Kotlin (`kotlin/`) | `main.kt`, `person.kt` + outputs (note: duplicate `person.kt.json`) | Coroutines, data classes, extensions. | -| Python (`python/`) | `main.py`, `calculator.py` + outputs | Functions, classes, comprehensions. | -| Rust (`rust/`) | `main.rs`, `point.rs` + outputs | Traits, structs, ownership patterns. | -| TS (`ts/`) | `main.ts`, `person.ts` + outputs | Interfaces, generics, type annotations. | - -No raw test runner files here—these artifacts are consumed by corresponding test modules like `tests/cpp.rs`, `tests/python.rs` (in `parsers/tests/`), which load/parse/validate them. - -**• Architecture:** -- **Layered Testing Pattern**: Fits into the AST module's parse → query → index pipeline (`src/ast/treesitter/parsers.rs` orchestrates language-specific parsers like `cpp.rs`, `rust.rs`). Tests validate the "parse_anything" contract from `ast_parse_anything.rs`. -- **Data-Driven Testing**: Each language mirrors production parser modules (`parsers/{lang}.rs`), using identical Tree-sitter grammars. Follows golden-file pattern: source → expected JSON/skeleton. -- **Relationships**: - - **Used by**: `treesitter/parsers/tests/{lang}.rs` (test runners), `skeletonizer.rs` (validates stripping logic), `ast_instance_structs.rs`/`file_ast_markup.rs` (AST node mapping). - - **Uses**: Tree-sitter crates (via `language_id.rs`), query files for decls/skeletons. - - **Integration**: Feeds into `ast_indexer_thread.rs`/`ast_db.rs` for workspace indexing; errors surface via `custom_error.rs`. Cross-references `alt_testsuite/` (more complex "torture" cases). -- **Patterns**: Repository pattern for test fixtures; language symmetry ensures uniform API (`structs.rs`). No runtime deps—pure validation. - -**• Key Symbols:** -(From consuming modules, inferred via structure:) -- `AstInstance`, `FileAstMarkup` (`ast_instance_structs.rs`): Structures validated against JSON. -- Parser fns: `parse_cpp()`, `skeletonize()` (`parsers/{lang}.rs`, `skeletonizer.rs`). -- Test utils: `load_test_case()`, `assert_ast_eq()` (in `parsers/tests/{lang}.rs`). -- Queries: Tree-sitter S-expression patterns for "decls", "skeleton" (in parser modules). - -**• Integration:** -- **Within AST**: Bottom of parse layer → top of indexing (`file_splitter.rs`, `chunk_utils.rs`). Builds upon `parse_common.rs`/`parse_python.rs` by providing concrete validation data. -- **Broader Agent**: Enables `@at_ast_definition`/`@at_ast_reference` (`at_commands/`), code completion RAG (`scratchpads/completon_rag.rs`), tools (`tool_ast_definition.rs`). -- **Cross-module**: Unlike general `tests/` (Python integration), this is Rust-unit focused. Complements `alt_testsuite/` (edge-case annotated files). Outputs feed VecDB indirectly via indexed skeletons. -- **Extension**: Easy to add languages (new dir + parser.rs + test.rs). Unlike VecDB tests (dynamic), these are static for parser fidelity. - -This directory embodies "test as documentation/spec"—files double as minimal repros for parser bugs, making the AST subsystem robust for multi-language agentic workflows. Compares to existing knowledge by providing the concrete data behind previously documented per-language test cases (Python/JS/etc.). diff --git a/.refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md b/.refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md deleted file mode 100644 index 488227842..000000000 --- a/.refact_knowledge/2025-12-17_044613_refactrefactagentguisrcfeaturesprovidersproviderfo.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: "`refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/components`" -created: 2025-12-17 -tags: ["architecture", "gui", "providers", "providermodelslist", "react-components", "refact-agent"] ---- - -### `refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/components` - -**Purpose** -This directory contains reusable React components for the **Provider Models List UI** within the Refact Agent's web-based GUI. It powers the model management interface in the `ProviderForm`, enabling users to view, add, edit, and configure AI models (e.g., from providers like OpenAI, Anthropic, Ollama) associated with a specific provider. The components focus on model cards, forms, badges, and dialogs, providing a modular, composable UI for dynamic model CRUD operations during provider setup/editing. This fits into the broader Providers feature, which abstracts AI backend selection (building on Rust engine's `caps/providers.rs` for capability-based model routing: `GlobalContext → Capabilities → Provider Selection → Model Inference`). - -**Files** -Despite the "empty" file read note, the structured project tree reveals these key components (all `.tsx` unless noted): -- **`AddModelButton.tsx`** - Trigger button for adding new models to the provider; handles dialog state and optimistic UI updates. -- **`CapabilityBadge.tsx`** - Visual badge displaying model capabilities (e.g., reasoning types, FIM support); extracts and renders human-readable labels from backend data. -- **`FormField.tsx`** - Generic form input field wrapper for model properties (e.g., model ID, API keys); supports validation and error states. -- **`FormSelect.tsx`** - Dropdown selector for model-related choices (e.g., selecting reasoning types or presets); integrates with form state. -- **`ModelCardPopup.tsx`** - Inline popup/edit dialog for individual model cards; handles detailed editing, deletion confirmation, and preview. -- **`index.ts`** - Barrel export re-exporting all components for easy imports in `ProviderModelsList.tsx`. - -Organization follows a flat, functional structure: action triggers (`AddModelButton`), display elements (`CapabilityBadge`, `ModelCardPopup`), and inputs (`FormField`, `FormSelect`). CSS modules (e.g., `ModelCard.module.css` in parent) suggest scoped styling. Naming uses descriptive PascalCase with "Form" prefix for inputs, emphasizing form-heavy interactions. - -**Architecture** -- **Single Responsibility & Composition**: Each file is a focused, stateless/presentational component. `ProviderModelsList.tsx` (parent) orchestrates them via hooks like `useModelDialogState.ts` from `./hooks`, composing lists of `ModelCard` → popups → forms. -- **State Management**: Relies on parent Redux slices (`providersSlice` implied via `useProviderForm.ts`) and local hooks for dialog state, optimistic updates, and mutations (e.g., `useUpdateProvider.ts`). Follows React Query/RTK Query patterns for backend sync (e.g., `useProvidersQuery`). -- **Data Flow**: Props-driven (model data from GraphQL/REST via `services/refact/providers.ts`); upward callbacks for mutations. Error boundaries via parent `ErrorState.tsx`. -- **Design Patterns**: - - **Compound Components**: `ModelCardPopup` + `FormField` form a mini-form system. - - **Render Props/Hooks**: Custom hooks in `./hooks` abstract dialog logic. - - **Layered**: Presentational layer only; business logic lifted to parent `ProviderForm`. -- Fits GUI's feature-slice architecture (`features/Providers/`): UI → hooks → Redux → services → Rust backend (`engine/src/caps/providers.rs`, `yaml_configs/default_providers/*.yaml`). - -**Key Symbols** -- **Components**: `AddModelButton`, `CapabilityBadge`, `FormField`, `FormSelect`, `ModelCardPopup`. -- **Hooks (inferred from parent `./hooks`)**: `useModelDialogState()` - Manages add/edit dialog visibility and temp state. -- **Props Patterns**: `{ model: ModelType, onUpdate: (model: ModelType) => void, capabilities: Capability[] }`; `extractHumanReadableReasoningType(reasoning: string)` utility from `./utils`. -- **Types**: Leverages shared `services/refact/types.ts` (e.g., `ProviderModel`, `Capability`); icons from `features/Providers/icons/` (e.g., `OpenAI.tsx`). -- **Constants**: Ties to `features/Providers/constants.ts` for provider/model presets. - -**Integration** -- **Used By**: `ProviderModelsList.tsx` (immediate parent) renders lists of these; aggregated in `ProviderForm.tsx` → `ProvidersView.tsx` → `Providers.tsx`. -- **Uses**: - - Parent hooks: `useProviderForm.ts`, `useProviderPreview.ts`. - - Utils: `./utils/extractHumanReadableReasoningType.ts` for badge text. - - Icons: `features/Providers/icons/iconsMap.tsx`. - - Services: Queries `useProvidersQuery()`, mutations via `useUpdateProvider.ts` → `services/refact/providers.ts` (GraphQL/REST to Rust `/v1/providers`). -- **Relationships**: - - **Inbound**: Model data from Redux (`providersSlice`) and RTK Query, synced with Rust `caps/providers.rs` (provider configs from `default_providers/*.yaml`). - - **Outbound**: Mutations propagate to backend, updating `GlobalContext` capabilities; used in `ChatForm/AgentCapabilities.tsx` for runtime tool/model selection. -- **Cross-Feature**: Links to `Integrations/` (provider configs enable integrations); `Chat/` consumes selected models. -- **Extension Points**: `CapabilityBadge` customizable via props; `FormField` generic for new model fields. Unlike simpler lists (e.g., `IntegrationsList`), this introduces dialog-driven editing for complex model configs (e.g., reasoning types, unlike flat `ConfiguredProvidersView`). Builds on core provider abstraction by providing fine-grained model UI, enabling self-hosted/custom setups (e.g., Ollama, LMStudio). - -This module exemplifies the GUI's "feature → form → list → components" nesting, prioritizing usability for provider/model management while abstracting Rust's capability layer. diff --git a/.refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md b/.refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md deleted file mode 100644 index 18bd8f436..000000000 --- a/.refact_knowledge/2025-12-17_044641_refactrefactagentguisrcfeaturesprovidersproviderfo.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: "`refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/hooks`" -created: 2025-12-17 -tags: ["architecture", "gui", "providers", "providermodelslist", "hooks", "react-hooks", "refact-agent"] ---- - -### `refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/hooks` - -**Purpose** -This directory contains custom React hooks that manage state and logic for the **Provider Models List** UI within the Refact Agent's web GUI. It handles dialog states, form interactions, and optimistic updates specifically for adding, editing, and deleting AI models associated with providers (e.g., configuring models like GPT-4 for OpenAI or Llama3 for Ollama). These hooks encapsulate complex UI behaviors like modal visibility, temporary model edits, and validation, keeping the presentational components in `./components` clean and reusable. This fits the Providers feature's role in abstracting Rust backend capabilities (`engine/src/caps/providers.rs`), enabling users to customize model routing for chat, completion, and agentic workflows via a declarative UI layer. - -**Files** -From project structure and sibling documentation: -- **`index.ts`** - Barrel export for all hooks (e.g., `export { useModelDialogState } from './useModelDialogState';`), enabling clean imports in `ProviderModelsList.tsx`. -- **`useModelDialogState.ts`** - Core hook managing add/edit/delete dialog lifecycle: tracks visibility, temporary model state (`tempModel: Partial`), editing mode (add vs. edit), and callbacks for save/cancel. Handles optimistic UI (local state before backend mutation) and error recovery. - -The directory follows the GUI's standard "hooks" convention: a single focused hook file + index export. Naming uses `use[Feature]State` pattern, emphasizing local dialog/form state over global Redux (which handles provider lists via `providersSlice`). - -**Architecture** -- **Hook Pattern & Single Responsibility**: `useModelDialogState` follows React's custom hook best practices—isolates mutable dialog logic (open/close, temp state, validation) from components. Returns an object like `{ isOpen, tempModel, openDialog(model?), closeDialog(), saveModel(), deleteModel() }` for ergonomic consumption. -- **State Management**: Local `useState`/`useReducer` for transient UI state (dialogs, drafts); integrates with parent `useProviderForm.ts` for form submission → RTK Query mutations (`useUpdateProvider`). No direct Redux dispatch—lifts state via callbacks to respect unidirectional data flow. -- **Data Flow**: Triggered by events from `./components` (e.g., `AddModelButton` clicks call `openDialog()`); effects sync with backend via `services/refact/providers.ts` → Rust `/v1/providers` endpoint. Error handling via try/catch + toast notifications (inferred from `ErrorState.tsx`). -- **Design Patterns**: - - **Custom Hook Abstraction**: Encapsulates dialog boilerplate (reducer for state transitions: IDLE → EDITING → SAVING → SUCCESS/ERROR). - - **Optimistic Updates**: Local mutations before API calls, with rollback on failure. - - **Layered Architecture**: Hooks (logic) → components (UI) → parent list (`ProviderModelsList.tsx`) → form (`ProviderForm.tsx`). Builds on Redux Toolkit Query for caching/query invalidation. -- Fits GUI's feature-slice organization (`features/Providers/ProviderForm/ProviderModelsList/`): hooks sit between presentational components and business logic, mirroring Rust's modular `caps/providers.rs` (provider → models → capabilities). - -**Key Symbols** -- **Hooks**: - - `useModelDialogState(props: { models: ProviderModel[], onSave: (model: ProviderModel) => void, onDelete: (id: string) => void })` → `{ isOpen: boolean, mode: 'add' | 'edit' | 'delete', tempModel: Partial, openDialog(model?: ProviderModel), closeDialog(), confirmDelete(), saveChanges() }`. -- **Internal State**: `dialogMode`, `tempModelId`, `isSubmitting` (loading states). -- **Types**: Relies on `services/refact/types.ts` (`ProviderModel: { id: string, name: string, reasoningType?: string, capabilities: string[] }`); utilities like `./utils/extractHumanReadableReasoningType`. -- **Dependencies**: `useCallback`, `useReducer` (state machine), RTK Query hooks from parent. No side effects beyond callbacks. - -**Integration** -- **Used By**: `./components` (e.g., `AddModelButton`, `ModelCardPopup` consume dialog state); orchestrated in `ProviderModelsList.tsx` alongside `ModelCard.tsx`. Flows up to `ProviderForm.tsx` → `ProvidersView.tsx`. -- **Uses**: - - Parent: `useProviderForm.ts` (form context), `useUpdateProvider.ts` (mutations). - - Sibling dirs: `./utils/*` (reasoning type formatting), `./components/*` (renders based on hook returns). - - Global: `useAppDispatch/useAppSelector` (via parents for `providersSlice`), `services/refact/providers.ts` (GraphQL/REST), icons from `features/Providers/icons/`. -- **Relationships**: - - **Inbound**: Model lists from RTK Query (`useProvidersQuery`), synced with Rust `yaml_configs/default_providers/*.yaml` and `GlobalContext`. - - **Outbound**: Model mutations → backend capabilities update → runtime effects in `ChatForm/AgentCapabilities.tsx` (tool/model selection). - - **Cross-Feature**: Enables `Integrations/` (model configs power integration tools); consumed by `Chat/` for dynamic provider routing. -- **Compares to Existing**: Unlike broader `useProviderForm.ts` (full provider CRUD), this is narrowly scoped to model dialogs—**builds upon provider list patterns from `ConfiguredProvidersView` by introducing inline editing modals**. Unlike flat lists (e.g., `IntegrationsList`), adds stateful dialogs for complex nested data (models-within-providers), supporting self-hosted extensibility (e.g., custom Ollama models). Introduces **hook-driven optimistic UI**, absent in simpler views like `ProviderPreview`. Extension point: Add new hooks for advanced features (e.g., `useModelValidation`). - -This hooks directory exemplifies the GUI's "logic extraction" principle, making model management declarative and testable while bridging UI to Rust's provider abstraction layer. - ---- -title: "`refact/refact-agent/gui/src/features/Providers/ProviderForm/ProviderModelsList/hooks`" -created: 2025-12-17 -tags: ["architecture", "gui", "providers", "providermodelslist", "hooks", "react-hooks", "refact-agent"] diff --git a/.refact_knowledge/trajectories/2025-12-17_044542_analysis.md b/.refact_knowledge/trajectories/2025-12-17_044542_analysis.md deleted file mode 100644 index 5aaf216f2..000000000 --- a/.refact_knowledge/trajectories/2025-12-17_044542_analysis.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: "" -created: 2025-12-17 -tags: ["trajectory"] ---- - - -**Chronological Analysis:** - -1. **Initial Setup (Compressed History)**: User requested Inline Edit (Ctrl+K) from AI_FEATURES_ROADMAP.md. Multiple iterations fixed flashing, key leakage, transparency, sizing, borders. Evolved from paint() recursion → JBPopup → Inlay API + Swing embedding → EditorTextField → final JBTextField + EditorCustomElementRenderer. - -2. **First Screenshot (Line 47 bleeding)**: Fixed with timer-based repositioning (16ms debounce), direct inlay.bounds query, reduced height (100px→92px), Graphics clipping. - -3. **Second Screenshot (JB Agent style request)**: User provided target design. Rewrote to 3-row layout (input → "Ask AI..." → Accept/Discard), vertical accent bar, unified background, transparent rows. - -4. **Third Screenshot + Multiple Editors Issue**: Fixed accent bar removal, subtle gray border, single inlay at selectionStart (not per-line). - -5. **Threading Error #1** (`removeSelection()`): Wrapped in `runReadAction{}`. - -6. **Threading Error #2** + UI Issues (cut buttons, invisible borders): Strategic planning identified **root causes** - missing ReadActions for ALL editor model reads, double borders, hardcoded heights. Implemented snapshot pattern (`SubmitSnapshot`), dynamic `getPreferredSize()`, single manual border painting. - -7. **Final Border Fix**: User confirmed "almost superb!" but borders invisible. Strategic planning → manual `paintBorder()` with `ColorUtil.mix(bg, fg, 0.22)` + alpha (140/120). - -8. **Latest Request** (`print what endpoint is supposed to be`): User asked about LSP integration. Found `/v1/code-edit` POST with `{"code", "instruction", "cursor_file", "cursor_line"} → {"modified_code"}`. - -**Technical Accuracy Check:** -✅ All threading fixed (single `WriteIntentReadAction.compute{}` snapshots) -✅ Dynamic sizing (no clipping across DPI/fonts) -✅ Single guaranteed-visible border (`paintBorder()` override) -✅ LSP endpoint confirmed (`/v1/code-edit`) -✅ Multi-caret safe (earliest selectionStart) -✅ Reentrancy guards (`AtomicBoolean`, `requestSeq`) - -**Completeness**: Feature 95% done. LSP endpoint exists but untested. Ctrl+K registration pending. - -**Most Recent**: User asked about LSP endpoint after border fix confirmation. - - - -1. **Primary Request and Intent:** - - Implement **Inline Edit (Ctrl+K)** feature from AI_FEATURES_ROADMAP.md - - **Core requirements**: Cursor position → 3-row JB Agent-style UI → AI code generation → diff preview → Accept/Reject - - **No selection required** (cursor-only works), single panel even with multi-line selection - - **Visual**: Vertical accent bar → unified editor-background → subtle gray border → dynamic height - - **Latest**: "print what endpoint is supposed to be" → confirm LSP integration - -2. **Key Technical Concepts:** - - **IntelliJ Inlay API**: `addBlockElement(offset, relatesToPreceding=false, showAbove=true, priority=1, renderer)` - - **EditorCustomElementRenderer**: `calcWidthInPixels()`, `calcHeightInPixels()`, `paint()` → Swing component embedding - - **Threading Model**: `WriteIntentReadAction.compute{}` snapshots, `WriteCommandAction` for edits - - **Component Embedding**: `editor.contentComponent.add(comp)`, `setComponentZOrder(0)`, timer-based repositioning (16ms) - - **Key Isolation**: `UIUtil.HIDE_EDITOR_FROM_DATA_CONTEXT_PROPERTY`, `DataProvider`, key consuming (Enter/ESC/Up/Down) - - **LSP Integration**: `POST /v1/code-edit` → `{"code": context, "instruction": prompt, "cursor_file": path, "cursor_line": N} → {"modified_code"}` - - **Diff Preview**: Reuse `ModeProvider.getDiffMode().actionPerformed(editor, newCode)` - - **Reentrancy**: `AtomicBoolean isProcessing`, `AtomicLong requestSeq` for stale response rejection - -3. **Files and Code Sections:** - - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditAction.kt`** (Entry point) - ```kotlin - class InlineEditAction : AnAction("Edit with Refact AI", ..., Resources.Icons.LOGO_RED_16x16) { - override fun actionPerformed(e: AnActionEvent) { - InlineEditManager.getInstance(editor).showInputPanel() - } - } - ``` - - - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditManager.kt`** (Core logic, 253 lines) - - **Why important**: Orchestrates inlay creation, LSP calls, diff preview - - **Key changes**: `SubmitSnapshot` data class, single `WriteIntentReadAction.compute{}` for all editor reads - ```kotlin - private data class SubmitSnapshot(val anchorOffset: Int, val anchorLine: Int, val filePath: String, val context: String) - - fun showInputPanel() { WriteIntentReadAction.compute { selectionStart ?: caret.offset → lineStartOffset } } - - private fun onSubmit(instruction: String) { - val snap = WriteIntentReadAction.compute { ... } // ALL editor reads here - lspCodeEdit(project, snap.context, instruction, snap.filePath, snap.anchorLine) - WriteCommandAction.runWriteCommandAction(project) { insertGeneratedCode(generatedCode, snap.anchorOffset) } - } - ``` - - - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditInputRenderer.kt`** (Inlay renderer, 199 lines) - - **Why important**: Positions Swing component over inlay, handles scroll/resize - ```kotlin - override fun calcHeightInPixels(inlay: Inlay<*>): Int = component?.preferredSize?.height ?: JBUI.scale(96) - override fun paint(inlay: Inlay<*>, g: Graphics, targetRegion: Rectangle, ...) { - g2.color = schemeBg; g2.fillRect(...) // Background only - if (!isComponentAttached) attachComponent() else scheduleReposition() - } - private val repositionTimer = Timer(16) { repositionComponent() } - ``` - - - **`src/main/kotlin/com/smallcloud/refactai/inline_edit/InlineEditInputComponent.kt`** (3-row Swing UI, 256 lines) - - **Why important**: JB Agent-style UI with input, buttons, focus isolation - ```kotlin - private val borderColor: Color // Mixed from scheme bg+fg with alpha - override fun paintBorder(g: Graphics) { - g2.color = borderColor - g2.draw(Rectangle2D.Float(0.5f, 0.5f, (width-1).toFloat(), (height-1).toFloat())) - } - override fun getPreferredSize(): Dimension { - val base = super.getPreferredSize() - val w = maxOf(editor.scrollingModel.visibleArea.width - JBUI.scale(80), JBUI.scale(400)) - return Dimension(w, base.height) - } - ``` - - - **`src/main/kotlin/com/smallcloud/refactai/lsp/LSPHelper.kt:188-231`** (LSP endpoint) - ```kotlin - fun lspCodeEdit(project: Project, code: String, instruction: String, filePath: String, cursorLine: Int): String { - val url = baseUrl.resolve("/v1/code-edit") - val data = mapOf("code" to code, "instruction" to instruction, "cursor_file" to filePath, "cursor_line" to cursorLine) - // → response.modifiedCode - } - ``` - -4. **Problem Solving:** - - **Flashing/jumping**: → Timer-based repositioning + direct `inlay.bounds` - - **Key leakage**: → `HIDE_EDITOR_FROM_DATA_CONTEXT_PROPERTY` + key consuming - - **Transparency/bleeding**: → Opaque backgrounds, `clipRect`, component fills inlay bounds - - **Threading crashes**: → Single `WriteIntentReadAction.compute{}` snapshots for ALL editor reads - - **Cut buttons**: → Dynamic `getPreferredSize()` (Swing computes natural height) - - **Invisible borders**: → Manual `paintBorder()` with `ColorUtil.mix(bgColor, fg, 0.22)` + alpha - - **Multiple panels**: → Single inlay at `selectionStart` (multi-caret safe) - -5. **Pending Tasks:** - - **None explicit**. LSP endpoint confirmed (`/v1/code-edit`). Feature visually complete. - -6. **Current Work:** - Implemented **guaranteed-visible border** via manual `paintBorder()` override in `InlineEditInputComponent.kt`. User confirmed "almost superb!" then asked "print what endpoint is supposed to be". Confirmed `POST /v1/code-edit` with request/response format. Build successful. - -7. **Optional Next Step:** - Test LSP integration with `./gradlew runIde` + verify `/v1/code-edit` returns valid `modified_code`. - -8. **Direct Quotes for Next Step Context:** - - User: "It almost superb! The one thing - borders are invisible!" → Fixed with manual border painting - - User: "print what endpoint is supposed to be" → Confirmed `/v1/code-edit` - - -Please, continue the conversation based on the provided summary diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index e69de29bb..000000000 From f05e8252ea1dbfc09387f15f53a8899e050164fc Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 04:53:11 +1030 Subject: [PATCH 4/8] refactor(agentic): improve commit message generator with conventional commits Replace the existing commit message prompts with comprehensive guidance following the Conventional Commits specification: - Add all standard commit types (feat, fix, refactor, perf, docs, etc.) - Enforce imperative mood and 50/72 character rules - Support scopes for better organization - Add breaking change detection with ! notation and BREAKING CHANGE footer - Include issue reference handling (Fixes #, Closes #, Refs #) - Provide clear examples for each type of commit - Add guidelines for synthesizing user intent with diff analysis --- .../src/agentic/generate_commit_message.rs | 307 ++++++++++++++---- 1 file changed, 242 insertions(+), 65 deletions(-) diff --git a/refact-agent/engine/src/agentic/generate_commit_message.rs b/refact-agent/engine/src/agentic/generate_commit_message.rs index 2b788d756..cefa981ab 100644 --- a/refact-agent/engine/src/agentic/generate_commit_message.rs +++ b/refact-agent/engine/src/agentic/generate_commit_message.rs @@ -11,20 +11,64 @@ use tokio::sync::RwLock as ARwLock; use tracing::warn; use crate::files_in_workspace::detect_vcs_for_a_file_path; -const DIFF_ONLY_PROMPT: &str = r#"Analyze the given diff and generate a clear and descriptive commit message that explains the purpose of the changes. Your commit message should convey *why* the changes were made, *how* they improve the code, or what features or fixes are implemented, rather than just restating *what* the changes are. Aim for an informative, concise summary that would be easy for others to understand when reviewing the commit history. +const DIFF_ONLY_PROMPT: &str = r#"Generate a commit message following the Conventional Commits specification. -# Steps -1. Analyze the code diff to understand the changes made. -2. Determine the functionality added or removed, and the reason for these adjustments. -3. Summarize the details of the change in an accurate and informative, yet concise way. -4. Structure the message in a way that starts with a short summary line, followed by optional details if the change is complex. +# Conventional Commits Format + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +## Commit Types (REQUIRED - choose exactly one) +- `feat`: New feature (correlates with MINOR in SemVer) +- `fix`: Bug fix (correlates with PATCH in SemVer) +- `refactor`: Code restructuring without changing behavior +- `perf`: Performance improvement +- `docs`: Documentation only changes +- `style`: Code style changes (formatting, whitespace, semicolons) +- `test`: Adding or correcting tests +- `build`: Changes to build system or dependencies +- `ci`: Changes to CI configuration +- `chore`: Maintenance tasks (tooling, configs, no production code change) +- `revert`: Reverting a previous commit + +## Rules + +### Subject Line (REQUIRED) +1. Format: `(): ` or `: ` +2. Use imperative mood ("add" not "added" or "adds") +3. Do NOT capitalize the first letter of description +4. Do NOT end with a period +5. Keep under 50 characters (hard limit: 72) +6. Scope is optional but recommended for larger projects + +### Body (OPTIONAL - use for complex changes) +1. Separate from subject with a blank line +2. Wrap at 72 characters +3. Explain WHAT and WHY, not HOW +4. Use bullet points for multiple items + +### Footer (OPTIONAL) +1. Reference issues: `Fixes #123`, `Closes #456`, `Refs #789` +2. Breaking changes: Start with `BREAKING CHANGE:` or add `!` after type +3. Co-authors: `Co-authored-by: Name ` + +## Breaking Changes +- Add `!` after type/scope: `feat!:` or `feat(api)!:` +- Or include `BREAKING CHANGE:` footer with explanation -# Output Format +# Steps -The output should be a single commit message in the following format: -- A **first line summarizing** the purpose of the change. This line should be concise. -- Optionally, include a **second paragraph** with *additional context* if the change is complex or otherwise needs further clarification. - (e.g., if there's a bug fix, mention what problem was fixed and why the change works.) +1. Analyze the diff to understand what changed +2. Determine the PRIMARY type of change (feat, fix, refactor, etc.) +3. Identify scope from affected files/modules (optional) +4. Write description in imperative mood explaining the intent +5. Add body only if the change is complex and needs explanation +6. Add footer for issue references or breaking changes if applicable # Examples @@ -32,23 +76,17 @@ The output should be a single commit message in the following format: ```diff - public class UserManager { - private final UserDAO userDAO; - + public class UserManager { + private final UserService userService; + private final NotificationService notificationService; - - public UserManager(UserDAO userDAO) { -- this.userDAO = userDAO; -+ this.userService = new UserService(); -+ this.notificationService = new NotificationService(); - } ``` -**Output (commit message)**: +**Output**: ``` -Refactor `UserManager` to use `UserService` and `NotificationService` +refactor(user): replace UserDAO with service-based architecture -Replaced `UserDAO` with `UserService` and introduced `NotificationService` to improve separation of concerns and make user management logic reusable and extendable. +Introduce UserService and NotificationService to improve separation of +concerns and make user management logic more reusable. ``` **Input (diff)**: @@ -61,39 +99,142 @@ Replaced `UserDAO` with `UserService` and introduced `NotificationService` to im + accessAllowed = age > 17; ``` -**Output (commit message)**: +**Output**: ``` -Simplify age check logic for accessing permissions by using a single expression +refactor: simplify age check with ternary expression ``` -# Notes -- Make sure the commit messages are descriptive enough to convey why the change is being made without being too verbose. -- If applicable, add `Fixes #` or other references to link the commit to specific tickets. -- Avoid wording: "Updated", "Modified", or "Changed" without explicitly stating *why*—focus on *intent*."#; +**Input (diff)**: +```diff ++ export async function fetchUserProfile(userId: string) { ++ const response = await api.get(`/users/${userId}`); ++ return response.data; ++ } +``` -const DIFF_WITH_USERS_TEXT_PROMPT: &str = r#"Generate a commit message using the diff and the provided initial commit message as a template for context. +**Output**: +``` +feat(api): add user profile fetch endpoint +``` -[Additional details as needed.] +**Input (diff)**: +```diff +- const timeout = 5000; ++ const timeout = 30000; +``` -# Steps +**Output**: +``` +fix(database): increase query timeout to prevent failures -1. Analyze the code diff to understand the changes made. -2. Review the user's initial commit message to understand the intent and use it as a contextual starting point. -3. Determine the functionality added or removed, and the reason for these adjustments. -4. Combine insights from the diff and user's initial commit message to generate a more descriptive and complete commit message. -5. Summarize the details of the change in an accurate and informative, yet concise way. -6. Structure the message in a way that starts with a short summary line, followed by optional details if the change is complex. +Extend timeout from 5s to 30s to resolve query failures during peak load. -# Output Format +Fixes #234 +``` + +**Input (breaking change)**: +```diff +- function getUser(id) { return users[id]; } ++ function getUser(id) { return { user: users[id], metadata: {} }; } +``` + +**Output**: +``` +feat(api)!: wrap user response in object with metadata + +BREAKING CHANGE: getUser() now returns { user, metadata } instead of +user directly. Update all callers to access .user property. +``` + +# Important Guidelines + +- Choose the MOST significant type if changes span multiple categories +- Be specific in the description - avoid vague terms like "update", "fix stuff" +- The subject should complete: "If applied, this commit will " +- One commit = one logical change (if diff has unrelated changes, note it) +- Scope should reflect the module, component, or area affected"#; + +const DIFF_WITH_USERS_TEXT_PROMPT: &str = r#"Generate a commit message following Conventional Commits, using the user's input as context for intent. + +# Conventional Commits Format + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +## Commit Types (REQUIRED - choose exactly one) +- `feat`: New feature (correlates with MINOR in SemVer) +- `fix`: Bug fix (correlates with PATCH in SemVer) +- `refactor`: Code restructuring without changing behavior +- `perf`: Performance improvement +- `docs`: Documentation only changes +- `style`: Code style changes (formatting, whitespace, semicolons) +- `test`: Adding or correcting tests +- `build`: Changes to build system or dependencies +- `ci`: Changes to CI configuration +- `chore`: Maintenance tasks (tooling, configs, no production code change) +- `revert`: Reverting a previous commit + +## Rules + +### Subject Line (REQUIRED) +1. Format: `(): ` or `: ` +2. Use imperative mood ("add" not "added" or "adds") +3. Do NOT capitalize the first letter of description +4. Do NOT end with a period +5. Keep under 50 characters (hard limit: 72) +6. Scope is optional but recommended for larger projects + +### Body (OPTIONAL - use for complex changes) +1. Separate from subject with a blank line +2. Wrap at 72 characters +3. Explain WHAT and WHY, not HOW +4. Use bullet points for multiple items + +### Footer (OPTIONAL) +1. Reference issues: `Fixes #123`, `Closes #456`, `Refs #789` +2. Breaking changes: Start with `BREAKING CHANGE:` or add `!` after type +3. Co-authors: `Co-authored-by: Name ` + +## Breaking Changes +- Add `!` after type/scope: `feat!:` or `feat(api)!:` +- Or include `BREAKING CHANGE:` footer with explanation + +# Steps -The output should be a single commit message in the following format: -- A **first line summarizing** the purpose of the change. This line should be concise. -- Optionally, include a **second paragraph** with *additional context* if the change is complex or otherwise needs further clarification. - (e.g., if there's a bug fix, mention what problem was fixed and why the change works.) +1. Analyze the user's initial commit message to understand their intent +2. Analyze the diff to understand the actual changes +3. Determine the correct type based on the nature of changes +4. Extract or infer a scope from user input or affected files +5. Synthesize user intent + diff analysis into a proper conventional commit +6. If user mentions an issue number, include it in the footer # Examples -**Input (initial commit message)**: +**Input (user's message)**: +``` +fix the login bug +``` + +**Input (diff)**: +```diff +- if (user.password === input) { ++ if (await bcrypt.compare(input, user.passwordHash)) { +``` + +**Output**: +``` +fix(auth): use bcrypt for secure password comparison + +Replace plaintext password comparison with bcrypt hash verification +to fix authentication vulnerability. +``` + +**Input (user's message)**: ``` Refactor UserManager to use services instead of DAOs ``` @@ -102,49 +243,85 @@ Refactor UserManager to use services instead of DAOs ```diff - public class UserManager { - private final UserDAO userDAO; - + public class UserManager { + private final UserService userService; + private final NotificationService notificationService; +``` + +**Output**: +``` +refactor(user): replace UserDAO with service-based architecture + +Introduce UserService and NotificationService to improve separation of +concerns and make user management logic more reusable. +``` + +**Input (user's message)**: +``` +added new endpoint for users #123 +``` - public UserManager(UserDAO userDAO) { -- this.userDAO = userDAO; -+ this.userService = new UserService(); -+ this.notificationService = new NotificationService(); - } +**Input (diff)**: +```diff ++ @GetMapping("/users/{id}/preferences") ++ public ResponseEntity getUserPreferences(@PathVariable Long id) { ++ return ResponseEntity.ok(userService.getPreferences(id)); ++ } ``` -**Output (commit message)**: +**Output**: ``` -Refactor `UserManager` to use `UserService` and `NotificationService` +feat(api): add user preferences endpoint -Replaced `UserDAO` with `UserService` and introduced `NotificationService` to improve separation of concerns and make user management logic reusable and extendable. +Refs #123 ``` -**Input (initial commit message)**: +**Input (user's message)**: ``` -Simplify age check logic +cleanup ``` **Input (diff)**: ```diff -- if (age > 17) { -- accessAllowed = true; -- } else { -- accessAllowed = false; -- } -+ accessAllowed = age > 17; +- // TODO: implement later +- // console.log("debug"); +- const unusedVar = 42; +``` + +**Output**: +``` +chore: remove dead code and debug artifacts +``` + +**Input (user's message)**: +``` +BREAKING: change API response format +``` + +**Input (diff)**: +```diff +- return user; ++ return { data: user, version: "2.0" }; ``` -**Output (commit message)**: +**Output**: ``` -Simplify age check logic for accessing permissions by using a single expression +feat(api)!: wrap responses in versioned data envelope + +BREAKING CHANGE: All API responses now return { data, version } object +instead of raw data. Clients must access response.data for the payload. ``` -# Notes -- Make sure the commit messages are descriptive enough to convey why the change is being made without being too verbose. -- If applicable, add `Fixes #` or other references to link the commit to specific tickets. -- Avoid wording: "Updated", "Modified", or "Changed" without explicitly stating *why*—focus on *intent*."#; +# Important Guidelines + +- Preserve the user's intent but format it correctly +- If user mentions "bug", "fix", "broken" → likely `fix` +- If user mentions "add", "new", "feature" → likely `feat` +- If user mentions "refactor", "restructure", "reorganize" → `refactor` +- If user mentions "clean", "remove unused" → likely `chore` or `refactor` +- Extract issue numbers (#123) from user text and move to footer +- The subject should complete: "If applied, this commit will " +- Don't just paraphrase the user - analyze the diff to add specificity"#; const N_CTX: usize = 32000; const TEMPERATURE: f32 = 0.5; From d249cd6641f03b772c840ebf003b2a5f70485337 Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 16:16:27 +1030 Subject: [PATCH 5/8] feat: knowledge graph system with subchat-powered metadata enrichment - Add petgraph-based knowledge graph (kg_structs, kg_builder, kg_query) - Extended frontmatter: UUID, kind, status, links, review_after, superseded_by - Subchat-powered create_knowledge: auto-enriches metadata, deprecates outdated docs - Graph-enhanced search: expands results via shared tags/files/entities - Weekly background cleanup: auto-archives old trajectories and deprecated docs - Entity extraction from backticked identifiers - Staleness detection: orphan refs, past review, deprecated ready to archive New files: - src/knowledge_graph/mod.rs - src/knowledge_graph/kg_structs.rs - src/knowledge_graph/kg_builder.rs - src/knowledge_graph/kg_query.rs - src/knowledge_graph/kg_staleness.rs - src/knowledge_graph/kg_subchat.rs - src/knowledge_graph/kg_cleanup.rs Modified: - Cargo.toml: added petgraph - memories.rs: new frontmatter format, deprecate/archive functions - tool_create_knowledge.rs: 2-phase subchat pipeline - tool_knowledge.rs: graph-enhanced search - vdb_markdown_splitter.rs: uses KnowledgeFrontmatter - background_tasks.rs: added cleanup task --- refact-agent/engine/Cargo.toml | 1 + refact-agent/engine/src/background_tasks.rs | 1 + .../engine/src/knowledge_graph/kg_builder.rs | 130 +++++++ .../engine/src/knowledge_graph/kg_cleanup.rs | 105 ++++++ .../engine/src/knowledge_graph/kg_query.rs | 124 +++++++ .../src/knowledge_graph/kg_staleness.rs | 99 ++++++ .../engine/src/knowledge_graph/kg_structs.rs | 333 ++++++++++++++++++ .../engine/src/knowledge_graph/kg_subchat.rs | 209 +++++++++++ .../engine/src/knowledge_graph/mod.rs | 10 + refact-agent/engine/src/main.rs | 1 + refact-agent/engine/src/memories.rs | 168 ++++++--- .../engine/src/tools/tool_create_knowledge.rs | 204 +++++++++-- .../engine/src/tools/tool_knowledge.rs | 58 ++- .../engine/src/vecdb/vdb_markdown_splitter.rs | 68 +--- 14 files changed, 1380 insertions(+), 131 deletions(-) create mode 100644 refact-agent/engine/src/knowledge_graph/kg_builder.rs create mode 100644 refact-agent/engine/src/knowledge_graph/kg_cleanup.rs create mode 100644 refact-agent/engine/src/knowledge_graph/kg_query.rs create mode 100644 refact-agent/engine/src/knowledge_graph/kg_staleness.rs create mode 100644 refact-agent/engine/src/knowledge_graph/kg_structs.rs create mode 100644 refact-agent/engine/src/knowledge_graph/kg_subchat.rs create mode 100644 refact-agent/engine/src/knowledge_graph/mod.rs diff --git a/refact-agent/engine/Cargo.toml b/refact-agent/engine/Cargo.toml index 2c10b352f..49158c5f3 100644 --- a/refact-agent/engine/Cargo.toml +++ b/refact-agent/engine/Cargo.toml @@ -99,6 +99,7 @@ url = "2.4.1" uuid = { version = "1", features = ["v4", "serde"] } walkdir = "2.3" which = "7.0.1" +petgraph = "0.6" zerocopy = "0.8.14" # There you can use a local copy diff --git a/refact-agent/engine/src/background_tasks.rs b/refact-agent/engine/src/background_tasks.rs index 18a821718..a537614e7 100644 --- a/refact-agent/engine/src/background_tasks.rs +++ b/refact-agent/engine/src/background_tasks.rs @@ -47,6 +47,7 @@ pub async fn start_background_tasks(gcx: Arc>, _config_di tokio::spawn(crate::vecdb::vdb_highlev::vecdb_background_reload(gcx.clone())), tokio::spawn(crate::integrations::sessions::remove_expired_sessions_background_task(gcx.clone())), tokio::spawn(crate::git::cleanup::git_shadow_cleanup_background_task(gcx.clone())), + tokio::spawn(crate::knowledge_graph::knowledge_cleanup_background_task(gcx.clone())), ]); let ast = gcx.clone().read().await.ast_service.clone(); if let Some(ast_service) = ast { diff --git a/refact-agent/engine/src/knowledge_graph/kg_builder.rs b/refact-agent/engine/src/knowledge_graph/kg_builder.rs new file mode 100644 index 000000000..c5c60a302 --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_builder.rs @@ -0,0 +1,130 @@ +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use regex::Regex; +use tokio::sync::RwLock as ARwLock; +use tracing::info; +use walkdir::WalkDir; + +use crate::file_filter::KNOWLEDGE_FOLDER_NAME; +use crate::files_correction::get_project_dirs; +use crate::files_in_workspace::get_file_text_from_memory_or_disk; +use crate::global_context::GlobalContext; + +use super::kg_structs::{KnowledgeDoc, KnowledgeFrontmatter, KnowledgeGraph}; + +fn extract_entities(content: &str) -> Vec { + let backtick_re = Regex::new(r"`([a-zA-Z_][a-zA-Z0-9_:]*(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)`").unwrap(); + let mut entities: HashSet = HashSet::new(); + + for caps in backtick_re.captures_iter(content) { + let entity = caps.get(1).unwrap().as_str().to_string(); + if entity.len() >= 3 && entity.len() <= 100 { + entities.insert(entity); + } + } + + entities.into_iter().collect() +} + +pub async fn build_knowledge_graph(gcx: Arc>) -> KnowledgeGraph { + let mut graph = KnowledgeGraph::new(); + + let project_dirs = get_project_dirs(gcx.clone()).await; + let knowledge_dirs: Vec = project_dirs.iter() + .map(|d| d.join(KNOWLEDGE_FOLDER_NAME)) + .filter(|d| d.exists()) + .collect(); + + if knowledge_dirs.is_empty() { + info!("knowledge_graph: no .refact_knowledge directories found"); + return graph; + } + + let workspace_files = collect_workspace_files(gcx.clone()).await; + let mut doc_count = 0; + + for knowledge_dir in knowledge_dirs { + for entry in WalkDir::new(&knowledge_dir) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if !path.is_file() { + continue; + } + + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); + if ext != "md" && ext != "mdx" { + continue; + } + + if path.to_string_lossy().contains("/archive/") { + continue; + } + + let path_buf = path.to_path_buf(); + let text = match get_file_text_from_memory_or_disk(gcx.clone(), &path_buf).await { + Ok(t) => t, + Err(_) => continue, + }; + + let (frontmatter, content_start) = KnowledgeFrontmatter::parse(&text); + let content = text[content_start..].to_string(); + let entities = extract_entities(&content); + + let mut validated_filenames = Vec::new(); + for filename in &frontmatter.filenames { + let exists = workspace_files.contains(filename); + if exists { + validated_filenames.push(filename.clone()); + } + graph.get_or_create_file(filename, exists); + } + + let doc = KnowledgeDoc { + path: path_buf, + frontmatter: KnowledgeFrontmatter { + filenames: validated_filenames, + ..frontmatter + }, + content, + entities, + }; + + graph.add_doc(doc); + doc_count += 1; + } + } + + graph.link_docs(); + info!("knowledge_graph: built with {} documents, {} tags, {} file refs, {} entities", + doc_count, + graph.tag_index.len(), + graph.file_index.len(), + graph.entity_index.len() + ); + + graph +} + +async fn collect_workspace_files(gcx: Arc>) -> HashSet { + let project_dirs = get_project_dirs(gcx.clone()).await; + let mut files = HashSet::new(); + + for dir in project_dirs { + let indexing = crate::files_blocklist::reload_indexing_everywhere_if_needed(gcx.clone()).await; + if let Ok(paths) = crate::files_in_workspace::ls_files(&*indexing, &dir, true) { + for path in paths { + if let Ok(rel) = path.strip_prefix(&dir) { + files.insert(rel.to_string_lossy().to_string()); + } + files.insert(path.to_string_lossy().to_string()); + } + } + } + + files +} + + diff --git a/refact-agent/engine/src/knowledge_graph/kg_cleanup.rs b/refact-agent/engine/src/knowledge_graph/kg_cleanup.rs new file mode 100644 index 000000000..fe03be16e --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_cleanup.rs @@ -0,0 +1,105 @@ +use std::sync::Arc; +use tokio::sync::RwLock as ARwLock; +use tokio::fs; +use tracing::{info, warn}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; + +use crate::global_context::GlobalContext; +use crate::memories::archive_document; +use super::kg_builder::build_knowledge_graph; + +const CLEANUP_INTERVAL_SECS: u64 = 7 * 24 * 60 * 60; +const TRAJECTORY_MAX_AGE_DAYS: i64 = 90; +const STALE_DOC_AGE_DAYS: i64 = 180; + +#[derive(Debug, Serialize, Deserialize, Default)] +struct CleanupState { + last_run: i64, +} + +async fn load_cleanup_state(gcx: Arc>) -> CleanupState { + let cache_dir = gcx.read().await.cache_dir.clone(); + let state_file = cache_dir.join("knowledge_cleanup_state.json"); + + match fs::read_to_string(&state_file).await { + Ok(content) => serde_json::from_str(&content).unwrap_or_default(), + Err(_) => CleanupState::default(), + } +} + +async fn save_cleanup_state(gcx: Arc>, state: &CleanupState) { + let cache_dir = gcx.read().await.cache_dir.clone(); + let state_file = cache_dir.join("knowledge_cleanup_state.json"); + + if let Ok(content) = serde_json::to_string(state) { + let _ = fs::write(&state_file, content).await; + } +} + +pub async fn knowledge_cleanup_background_task(gcx: Arc>) { + loop { + let state = load_cleanup_state(gcx.clone()).await; + let now = Utc::now().timestamp(); + + if now - state.last_run >= CLEANUP_INTERVAL_SECS as i64 { + info!("knowledge_cleanup: running weekly cleanup"); + + match run_cleanup(gcx.clone()).await { + Ok(report) => { + info!("knowledge_cleanup: completed - archived {} trajectories, {} deprecated docs, {} orphan warnings", + report.archived_trajectories, + report.archived_deprecated, + report.orphan_warnings, + ); + } + Err(e) => { + warn!("knowledge_cleanup: failed - {}", e); + } + } + + let new_state = CleanupState { last_run: now }; + save_cleanup_state(gcx.clone(), &new_state).await; + } + + tokio::time::sleep(tokio::time::Duration::from_secs(24 * 60 * 60)).await; + } +} + +#[derive(Debug, Default)] +struct CleanupReport { + archived_trajectories: usize, + archived_deprecated: usize, + orphan_warnings: usize, +} + +async fn run_cleanup(gcx: Arc>) -> Result { + let kg = build_knowledge_graph(gcx.clone()).await; + let staleness = kg.check_staleness(STALE_DOC_AGE_DAYS, TRAJECTORY_MAX_AGE_DAYS); + let mut report = CleanupReport::default(); + + for path in staleness.stale_trajectories { + match archive_document(gcx.clone(), &path).await { + Ok(_) => report.archived_trajectories += 1, + Err(e) => warn!("Failed to archive trajectory {}: {}", path.display(), e), + } + } + + for path in staleness.deprecated_ready_to_archive { + match archive_document(gcx.clone(), &path).await { + Ok(_) => report.archived_deprecated += 1, + Err(e) => warn!("Failed to archive deprecated doc {}: {}", path.display(), e), + } + } + + report.orphan_warnings = staleness.orphan_file_refs.len(); + for (path, missing_files) in &staleness.orphan_file_refs { + info!("knowledge_cleanup: {} references missing files: {:?}", path.display(), missing_files); + } + + for path in &staleness.past_review { + info!("knowledge_cleanup: {} is past review date", path.display()); + } + + Ok(report) +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_query.rs b/refact-agent/engine/src/knowledge_graph/kg_query.rs new file mode 100644 index 000000000..f795cd3f3 --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_query.rs @@ -0,0 +1,124 @@ +use std::collections::{HashMap, HashSet}; + +use super::kg_structs::{KnowledgeDoc, KnowledgeGraph}; + +struct RelatedDoc { + id: String, + score: f64, +} + +impl KnowledgeGraph { + fn find_related(&self, doc_id: &str, max_results: usize) -> Vec { + let Some(doc) = self.docs.get(doc_id) else { + return vec![]; + }; + + let mut scores: HashMap = HashMap::new(); + + for tag in &doc.frontmatter.tags { + for related_id in self.docs_with_tag(tag) { + if related_id != doc_id { + *scores.entry(related_id).or_insert(0.0) += 1.0; + } + } + } + + for filename in &doc.frontmatter.filenames { + for related_id in self.docs_referencing_file(filename) { + if related_id != doc_id { + *scores.entry(related_id).or_insert(0.0) += 2.0; + } + } + } + + for entity in &doc.entities { + for related_id in self.docs_mentioning_entity(entity) { + if related_id != doc_id { + *scores.entry(related_id).or_insert(0.0) += 1.5; + } + } + } + + let mut results: Vec = scores.into_iter() + .filter(|(id, _)| self.docs.get(id).map(|d| d.frontmatter.is_active()).unwrap_or(false)) + .map(|(id, score)| RelatedDoc { id, score }) + .collect(); + + results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + results.truncate(max_results); + results + } + + pub fn expand_search_results(&self, initial_doc_ids: &[String], max_expansion: usize) -> Vec { + let mut all_ids: HashSet = initial_doc_ids.iter().cloned().collect(); + let mut expanded: Vec = vec![]; + + for doc_id in initial_doc_ids { + let related = self.find_related(doc_id, max_expansion); + for rel in related { + if !all_ids.contains(&rel.id) { + all_ids.insert(rel.id.clone()); + expanded.push(rel.id); + } + if expanded.len() >= max_expansion { + break; + } + } + if expanded.len() >= max_expansion { + break; + } + } + + expanded + } + + fn find_similar_docs(&self, tags: &[String], filenames: &[String], entities: &[String]) -> Vec<(String, f64)> { + let mut scores: HashMap = HashMap::new(); + + for tag in tags { + for id in self.docs_with_tag(tag) { + *scores.entry(id).or_insert(0.0) += 1.0; + } + } + + for filename in filenames { + for id in self.docs_referencing_file(filename) { + *scores.entry(id).or_insert(0.0) += 2.0; + } + } + + for entity in entities { + for id in self.docs_mentioning_entity(entity) { + *scores.entry(id).or_insert(0.0) += 1.5; + } + } + + let mut results: Vec<_> = scores.into_iter() + .filter(|(id, _)| self.docs.get(id).map(|d| d.frontmatter.is_active()).unwrap_or(false)) + .collect(); + + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + results + } + + pub fn get_deprecation_candidates(&self, new_doc_tags: &[String], new_doc_filenames: &[String], new_doc_entities: &[String], exclude_id: Option<&str>) -> Vec<&KnowledgeDoc> { + let similar = self.find_similar_docs(new_doc_tags, new_doc_filenames, new_doc_entities); + + similar.into_iter() + .filter(|(id, score)| { + *score >= 2.0 && exclude_id.map(|e| e != id).unwrap_or(true) + }) + .filter_map(|(id, _)| { + let doc = self.docs.get(&id)?; + if doc.frontmatter.is_deprecated() || doc.frontmatter.is_archived() { + return None; + } + if doc.frontmatter.kind_or_default() == "trajectory" { + return None; + } + Some(doc) + }) + .take(10) + .collect() + } +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_staleness.rs b/refact-agent/engine/src/knowledge_graph/kg_staleness.rs new file mode 100644 index 000000000..97fe6e89d --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_staleness.rs @@ -0,0 +1,99 @@ +use std::collections::HashSet; +use std::path::PathBuf; +use chrono::{NaiveDate, Utc}; + +use super::kg_structs::KnowledgeGraph; + +#[derive(Debug, Default)] +pub struct StalenessReport { + pub orphan_file_refs: Vec<(PathBuf, Vec)>, + pub orphan_docs: Vec, + pub stale_by_age: Vec<(PathBuf, i64)>, + pub past_review: Vec, + pub deprecated_ready_to_archive: Vec, + pub stale_trajectories: Vec, +} + +impl KnowledgeGraph { + pub fn check_staleness(&self, max_age_days: i64, trajectory_max_age_days: i64) -> StalenessReport { + let mut report = StalenessReport::default(); + let today = Utc::now().date_naive(); + + for doc in self.docs.values() { + let kind = doc.frontmatter.kind_or_default(); + + if let Some(created) = &doc.frontmatter.created { + if let Ok(created_date) = NaiveDate::parse_from_str(created, "%Y-%m-%d") { + let age_days = (today - created_date).num_days(); + + if kind == "trajectory" && age_days > trajectory_max_age_days { + report.stale_trajectories.push(doc.path.clone()); + continue; + } + + if age_days > max_age_days && doc.frontmatter.is_active() { + report.stale_by_age.push((doc.path.clone(), age_days)); + } + } + } + + if let Some(review_after) = &doc.frontmatter.review_after { + if let Ok(review_date) = NaiveDate::parse_from_str(review_after, "%Y-%m-%d") { + if today > review_date && doc.frontmatter.is_active() { + report.past_review.push(doc.path.clone()); + } + } + } + + if doc.frontmatter.is_deprecated() { + if let Some(deprecated_at) = &doc.frontmatter.deprecated_at { + if let Ok(deprecated_date) = NaiveDate::parse_from_str(deprecated_at, "%Y-%m-%d") { + let days_deprecated = (today - deprecated_date).num_days(); + if days_deprecated > 60 { + report.deprecated_ready_to_archive.push(doc.path.clone()); + } + } + } + } + + let missing_files: Vec = doc.frontmatter.filenames.iter() + .filter(|f| { + self.file_index.get(*f) + .and_then(|idx| self.graph.node_weight(*idx)) + .map(|node| { + if let super::kg_structs::KgNode::FileRef { exists, .. } = node { + !exists + } else { + false + } + }) + .unwrap_or(true) + }) + .cloned() + .collect(); + + if !missing_files.is_empty() && doc.frontmatter.kind_or_default() == "code" { + report.orphan_file_refs.push((doc.path.clone(), missing_files)); + } + } + + let docs_with_links: HashSet = self.docs.values() + .flat_map(|d| d.frontmatter.links.iter()) + .filter_map(|link| self.docs.get(link)) + .map(|d| d.path.clone()) + .collect(); + + for doc in self.docs.values() { + if doc.frontmatter.tags.is_empty() + && doc.frontmatter.filenames.is_empty() + && doc.entities.is_empty() + && !docs_with_links.contains(&doc.path) + && doc.frontmatter.kind_or_default() != "trajectory" + { + report.orphan_docs.push(doc.path.clone()); + } + } + + report + } +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_structs.rs b/refact-agent/engine/src/knowledge_graph/kg_structs.rs new file mode 100644 index 000000000..dab6e6dc5 --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_structs.rs @@ -0,0 +1,333 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use serde::{Deserialize, Serialize}; +use petgraph::graph::{DiGraph, NodeIndex}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct KnowledgeFrontmatter { + #[serde(default)] + pub id: Option, + #[serde(default)] + pub title: Option, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub created: Option, + #[serde(default)] + pub updated: Option, + #[serde(default)] + pub filenames: Vec, + #[serde(default)] + pub links: Vec, + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub status: Option, + #[serde(default)] + pub superseded_by: Option, + #[serde(default)] + pub deprecated_at: Option, + #[serde(default)] + pub review_after: Option, +} + +impl KnowledgeFrontmatter { + pub fn parse(content: &str) -> (Self, usize) { + if !content.starts_with("---") { + return (Self::default(), 0); + } + + let rest = &content[3..]; + let end_marker = rest.find("\n---"); + let Some(end_idx) = end_marker else { + return (Self::default(), 0); + }; + + let yaml_content = &rest[..end_idx]; + let mut end_offset = 3 + end_idx + 4; + if content.len() > end_offset && content.as_bytes().get(end_offset) == Some(&b'\n') { + end_offset += 1; + } + + match serde_yaml::from_str::(yaml_content) { + Ok(fm) => (fm, end_offset), + Err(_) => (Self::default(), 0), + } + } + + pub fn to_yaml(&self) -> String { + let mut lines = vec!["---".to_string()]; + + if let Some(id) = &self.id { + lines.push(format!("id: \"{}\"", id)); + } + if let Some(title) = &self.title { + lines.push(format!("title: \"{}\"", title.replace('"', "\\\""))); + } + if let Some(kind) = &self.kind { + lines.push(format!("kind: {}", kind)); + } + if let Some(created) = &self.created { + lines.push(format!("created: {}", created)); + } + if let Some(updated) = &self.updated { + lines.push(format!("updated: {}", updated)); + } + if let Some(review_after) = &self.review_after { + lines.push(format!("review_after: {}", review_after)); + } + if let Some(status) = &self.status { + lines.push(format!("status: {}", status)); + } + if !self.tags.is_empty() { + let tags_str = self.tags.iter() + .map(|t| format!("\"{}\"", t)) + .collect::>() + .join(", "); + lines.push(format!("tags: [{}]", tags_str)); + } + if !self.filenames.is_empty() { + let files_str = self.filenames.iter() + .map(|f| format!("\"{}\"", f)) + .collect::>() + .join(", "); + lines.push(format!("filenames: [{}]", files_str)); + } + if !self.links.is_empty() { + let links_str = self.links.iter() + .map(|l| format!("\"{}\"", l)) + .collect::>() + .join(", "); + lines.push(format!("links: [{}]", links_str)); + } + if let Some(superseded_by) = &self.superseded_by { + lines.push(format!("superseded_by: \"{}\"", superseded_by)); + } + if let Some(deprecated_at) = &self.deprecated_at { + lines.push(format!("deprecated_at: {}", deprecated_at)); + } + + lines.push("---".to_string()); + lines.join("\n") + } + + pub fn is_active(&self) -> bool { + self.status.as_deref().unwrap_or("active") == "active" + } + + pub fn is_deprecated(&self) -> bool { + self.status.as_deref() == Some("deprecated") + } + + pub fn is_archived(&self) -> bool { + self.status.as_deref() == Some("archived") + } + + pub fn kind_or_default(&self) -> &str { + self.kind.as_deref().unwrap_or(if self.filenames.is_empty() { "domain" } else { "code" }) + } +} + +#[derive(Debug, Clone)] +pub struct KnowledgeDoc { + pub path: PathBuf, + pub frontmatter: KnowledgeFrontmatter, + pub content: String, + pub entities: Vec, +} + +#[derive(Debug, Clone)] +pub enum KgNode { + Doc { id: String }, + Tag, + FileRef { exists: bool }, + Entity, +} + +#[derive(Debug, Clone)] +pub enum KgEdge { + TaggedWith, + ReferencesFile, + LinksTo, + Mentions, + SupersededBy, +} + +pub struct KnowledgeGraph { + pub graph: DiGraph, + pub doc_index: HashMap, + pub doc_path_index: HashMap, + pub tag_index: HashMap, + pub file_index: HashMap, + pub entity_index: HashMap, + pub docs: HashMap, +} + +impl Default for KnowledgeGraph { + fn default() -> Self { + Self::new() + } +} + +impl KnowledgeGraph { + pub fn new() -> Self { + Self { + graph: DiGraph::new(), + doc_index: HashMap::new(), + doc_path_index: HashMap::new(), + tag_index: HashMap::new(), + file_index: HashMap::new(), + entity_index: HashMap::new(), + docs: HashMap::new(), + } + } + + pub fn get_or_create_tag(&mut self, name: &str) -> NodeIndex { + let normalized = name.to_lowercase().trim().to_string(); + if let Some(&idx) = self.tag_index.get(&normalized) { + return idx; + } + let idx = self.graph.add_node(KgNode::Tag); + self.tag_index.insert(normalized, idx); + idx + } + + pub fn get_or_create_file(&mut self, path: &str, exists: bool) -> NodeIndex { + if let Some(&idx) = self.file_index.get(path) { + return idx; + } + let idx = self.graph.add_node(KgNode::FileRef { exists }); + self.file_index.insert(path.to_string(), idx); + idx + } + + pub fn get_or_create_entity(&mut self, name: &str) -> NodeIndex { + if let Some(&idx) = self.entity_index.get(name) { + return idx; + } + let idx = self.graph.add_node(KgNode::Entity); + self.entity_index.insert(name.to_string(), idx); + idx + } + + pub fn add_doc(&mut self, doc: KnowledgeDoc) -> NodeIndex { + let id = doc.frontmatter.id.clone().unwrap_or_else(|| doc.path.to_string_lossy().to_string()); + let path = doc.path.clone(); + + let doc_idx = self.graph.add_node(KgNode::Doc { id: id.clone() }); + self.doc_index.insert(id.clone(), doc_idx); + self.doc_path_index.insert(path, doc_idx); + + for tag in &doc.frontmatter.tags { + let tag_idx = self.get_or_create_tag(tag); + self.graph.add_edge(doc_idx, tag_idx, KgEdge::TaggedWith); + } + + for filename in &doc.frontmatter.filenames { + let file_idx = self.get_or_create_file(filename, true); + self.graph.add_edge(doc_idx, file_idx, KgEdge::ReferencesFile); + } + + for entity in &doc.entities { + let entity_idx = self.get_or_create_entity(entity); + self.graph.add_edge(doc_idx, entity_idx, KgEdge::Mentions); + } + + self.docs.insert(id, doc); + doc_idx + } + + pub fn link_docs(&mut self) { + let links: Vec<(String, String)> = self.docs.iter() + .flat_map(|(id, doc)| { + doc.frontmatter.links.iter().map(|link| (id.clone(), link.clone())).collect::>() + }) + .collect(); + + for (from_id, to_id) in links { + if let (Some(&from_idx), Some(&to_idx)) = (self.doc_index.get(&from_id), self.doc_index.get(&to_id)) { + self.graph.add_edge(from_idx, to_idx, KgEdge::LinksTo); + } + } + + let supersedes: Vec<(String, String)> = self.docs.iter() + .filter_map(|(id, doc)| { + doc.frontmatter.superseded_by.as_ref().map(|s| (id.clone(), s.clone())) + }) + .collect(); + + for (old_id, new_id) in supersedes { + if let (Some(&old_idx), Some(&new_idx)) = (self.doc_index.get(&old_id), self.doc_index.get(&new_id)) { + self.graph.add_edge(old_idx, new_idx, KgEdge::SupersededBy); + } + } + } + + pub fn get_doc_by_id(&self, id: &str) -> Option<&KnowledgeDoc> { + self.docs.get(id) + } + + pub fn get_doc_by_path(&self, path: &PathBuf) -> Option<&KnowledgeDoc> { + self.doc_path_index.get(path) + .and_then(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(*idx) { + self.docs.get(id) + } else { + None + } + }) + } + + pub fn active_docs(&self) -> impl Iterator { + self.docs.values().filter(|d| d.frontmatter.is_active()) + } + + pub fn docs_with_tag(&self, tag: &str) -> HashSet { + let normalized = tag.to_lowercase(); + let Some(&tag_idx) = self.tag_index.get(&normalized) else { + return HashSet::new(); + }; + + self.graph.neighbors_directed(tag_idx, petgraph::Direction::Incoming) + .filter_map(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(idx) { + Some(id.clone()) + } else { + None + } + }) + .collect() + } + + pub fn docs_referencing_file(&self, file_path: &str) -> HashSet { + let Some(&file_idx) = self.file_index.get(file_path) else { + return HashSet::new(); + }; + + self.graph.neighbors_directed(file_idx, petgraph::Direction::Incoming) + .filter_map(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(idx) { + Some(id.clone()) + } else { + None + } + }) + .collect() + } + + pub fn docs_mentioning_entity(&self, entity: &str) -> HashSet { + let Some(&entity_idx) = self.entity_index.get(entity) else { + return HashSet::new(); + }; + + self.graph.neighbors_directed(entity_idx, petgraph::Direction::Incoming) + .filter_map(|idx| { + if let Some(KgNode::Doc { id, .. }) = self.graph.node_weight(idx) { + Some(id.clone()) + } else { + None + } + }) + .collect() + } +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_subchat.rs b/refact-agent/engine/src/knowledge_graph/kg_subchat.rs new file mode 100644 index 000000000..553ad427d --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/kg_subchat.rs @@ -0,0 +1,209 @@ +use std::sync::Arc; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex as AMutex; + +use crate::at_commands::at_commands::AtCommandsContext; +use crate::call_validation::ChatMessage; +use crate::subchat::subchat_single; + +use super::kg_structs::KnowledgeDoc; + +#[derive(Debug, Serialize, Deserialize)] +pub struct EnrichmentResult { + #[serde(default)] + pub title: Option, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub filenames: Vec, + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub links: Vec, + #[serde(default)] + pub review_after_days: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeprecationDecision { + #[serde(default)] + pub target_id: String, + #[serde(default)] + pub reason: String, + #[serde(default)] + pub confidence: f64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeprecationResult { + #[serde(default)] + pub deprecate: Vec, + #[serde(default)] + pub keep: Vec, +} + +const ENRICHMENT_PROMPT: &str = r#"Analyze the following knowledge content and extract metadata. + +CONTENT: +{content} + +EXTRACTED ENTITIES (backticked identifiers found): +{entities} + +CANDIDATE FILES (from workspace, pick only relevant ones): +{candidate_files} + +CANDIDATE RELATED DOCS (from knowledge base, pick only relevant ones): +{candidate_docs} + +Return a JSON object with: +- title: a concise title for this knowledge (max 80 chars) +- tags: array of relevant tags (lowercase, max 8 tags) +- filenames: array of file paths this knowledge relates to (only from candidates) +- kind: one of "code", "decision", "domain", "process" (based on content) +- links: array of related doc IDs (only from candidates) +- review_after_days: suggested review period (90 for code/decision, 180 for domain) + +JSON only, no explanation:"#; + +const DEPRECATION_PROMPT: &str = r#"A new knowledge document was created. Determine if any existing documents should be deprecated. + +NEW DOCUMENT: +Title: {new_title} +Tags: {new_tags} +Files: {new_files} +Content snippet: {new_snippet} + +CANDIDATE DOCUMENTS TO POTENTIALLY DEPRECATE: +{candidates} + +For each candidate, decide if it should be deprecated because: +- It covers the same topic and the new doc is more complete/updated +- It references the same files with outdated information +- It's a duplicate or near-duplicate + +Return JSON: +{{ + "deprecate": [ + {{"target_id": "...", "reason": "...", "confidence": 0.0-1.0}} + ], + "keep": ["id1", "id2"] +}} + +Only deprecate with confidence >= 0.75. JSON only:"#; + +pub async fn enrich_knowledge_metadata( + ccx: Arc>, + model_id: &str, + content: &str, + entities: &[String], + candidate_files: &[String], + candidate_docs: &[(String, String)], +) -> Result { + let entities_str = entities.join(", "); + let files_str = candidate_files.iter().take(20).cloned().collect::>().join("\n"); + let docs_str = candidate_docs.iter() + .take(10) + .map(|(id, title)| format!("- {}: {}", id, title)) + .collect::>() + .join("\n"); + + let prompt = ENRICHMENT_PROMPT + .replace("{content}", &content.chars().take(2000).collect::()) + .replace("{entities}", &entities_str) + .replace("{candidate_files}", &files_str) + .replace("{candidate_docs}", &docs_str); + + let messages = vec![ChatMessage::new("user".to_string(), prompt)]; + + let results = subchat_single( + ccx, + model_id, + messages, + Some(vec![]), + Some("none".to_string()), + false, + Some(0.0), + Some(1024), + 1, + None, + false, + None, + None, + None, + ).await?; + + let response = results.get(0) + .and_then(|msgs| msgs.last()) + .map(|m| m.content.content_text_only()) + .unwrap_or_default(); + + let json_start = response.find('{').unwrap_or(0); + let json_end = response.rfind('}').map(|i| i + 1).unwrap_or(response.len()); + let json_str = &response[json_start..json_end]; + + serde_json::from_str(json_str).map_err(|e| format!("Failed to parse enrichment JSON: {}", e)) +} + +pub async fn check_deprecation( + ccx: Arc>, + model_id: &str, + new_doc_title: &str, + new_doc_tags: &[String], + new_doc_files: &[String], + new_doc_snippet: &str, + candidates: &[&KnowledgeDoc], +) -> Result { + if candidates.is_empty() { + return Ok(DeprecationResult { deprecate: vec![], keep: vec![] }); + } + + let candidates_str = candidates.iter() + .map(|doc| { + let id = doc.frontmatter.id.clone().unwrap_or_else(|| doc.path.to_string_lossy().to_string()); + let title = doc.frontmatter.title.clone().unwrap_or_default(); + let tags = doc.frontmatter.tags.join(", "); + let files = doc.frontmatter.filenames.join(", "); + let snippet: String = doc.content.chars().take(300).collect(); + format!("ID: {}\nTitle: {}\nTags: {}\nFiles: {}\nSnippet: {}\n---", id, title, tags, files, snippet) + }) + .collect::>() + .join("\n"); + + let prompt = DEPRECATION_PROMPT + .replace("{new_title}", new_doc_title) + .replace("{new_tags}", &new_doc_tags.join(", ")) + .replace("{new_files}", &new_doc_files.join(", ")) + .replace("{new_snippet}", &new_doc_snippet.chars().take(500).collect::()) + .replace("{candidates}", &candidates_str); + + let messages = vec![ChatMessage::new("user".to_string(), prompt)]; + + let results = subchat_single( + ccx, + model_id, + messages, + Some(vec![]), + Some("none".to_string()), + false, + Some(0.0), + Some(1024), + 1, + None, + false, + None, + None, + None, + ).await?; + + let response = results.get(0) + .and_then(|msgs| msgs.last()) + .map(|m| m.content.content_text_only()) + .unwrap_or_default(); + + let json_start = response.find('{').unwrap_or(0); + let json_end = response.rfind('}').map(|i| i + 1).unwrap_or(response.len()); + let json_str = &response[json_start..json_end]; + + serde_json::from_str(json_str).map_err(|e| format!("Failed to parse deprecation JSON: {}", e)) +} diff --git a/refact-agent/engine/src/knowledge_graph/mod.rs b/refact-agent/engine/src/knowledge_graph/mod.rs new file mode 100644 index 000000000..bd434b60b --- /dev/null +++ b/refact-agent/engine/src/knowledge_graph/mod.rs @@ -0,0 +1,10 @@ +pub mod kg_structs; +pub mod kg_builder; +pub mod kg_query; +pub mod kg_staleness; +pub mod kg_subchat; +pub mod kg_cleanup; + +pub use kg_structs::KnowledgeFrontmatter; +pub use kg_builder::build_knowledge_graph; +pub use kg_cleanup::knowledge_cleanup_background_task; diff --git a/refact-agent/engine/src/main.rs b/refact-agent/engine/src/main.rs index d1f6f4667..70fdab5d4 100644 --- a/refact-agent/engine/src/main.rs +++ b/refact-agent/engine/src/main.rs @@ -65,6 +65,7 @@ mod git; mod agentic; mod memories; mod files_correction_cache; +mod knowledge_graph; pub mod constants; #[tokio::main] diff --git a/refact-agent/engine/src/memories.rs b/refact-agent/engine/src/memories.rs index d99b3e59d..c2c888d7b 100644 --- a/refact-agent/engine/src/memories.rs +++ b/refact-agent/engine/src/memories.rs @@ -1,17 +1,18 @@ use std::path::PathBuf; use std::sync::Arc; -use chrono::Local; +use chrono::{Local, Duration}; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock as ARwLock; use tokio::fs; use tracing::info; +use uuid::Uuid; use walkdir::WalkDir; use crate::file_filter::KNOWLEDGE_FOLDER_NAME; use crate::files_correction::get_project_dirs; use crate::files_in_workspace::get_file_text_from_memory_or_disk; use crate::global_context::GlobalContext; -use crate::vecdb::vdb_markdown_splitter::MarkdownFrontmatter; +use crate::knowledge_graph::kg_structs::KnowledgeFrontmatter; use crate::vecdb::vdb_structs::VecdbSearch; #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq)] @@ -23,6 +24,7 @@ pub struct MemoRecord { pub line_range: Option<(u64, u64)>, pub title: Option, pub created: Option, + pub kind: Option, } fn generate_slug(content: &str) -> String { @@ -44,38 +46,44 @@ fn generate_slug(content: &str) -> String { fn generate_filename(content: &str) -> String { let timestamp = Local::now().format("%Y-%m-%d_%H%M%S").to_string(); let slug = generate_slug(content); + let short_uuid = &Uuid::new_v4().to_string()[..8]; if slug.is_empty() { - format!("{}_knowledge.md", timestamp) + format!("{}_{}_knowledge.md", timestamp, short_uuid) } else { - format!("{}_{}.md", timestamp, slug) + format!("{}_{}_{}.md", timestamp, short_uuid, slug) } } -fn create_markdown_content(tags: &[String], filenames: &[String], content: &str) -> String { - let now = Local::now().format("%Y-%m-%d").to_string(); - let title = content.lines().next().unwrap_or("Knowledge Entry"); - let tags_str = tags.iter().map(|t| format!("\"{}\"", t)).collect::>().join(", "); - let filenames_str = if filenames.is_empty() { - String::new() - } else { - format!("\nfilenames: [{}]", filenames.iter().map(|f| format!("\"{}\"", f)).collect::>().join(", ")) +pub fn create_frontmatter( + title: Option<&str>, + tags: &[String], + filenames: &[String], + links: &[String], + kind: &str, +) -> KnowledgeFrontmatter { + let now = Local::now(); + let created = now.format("%Y-%m-%d").to_string(); + let review_days = match kind { + "trajectory" => 90, + "preference" => 365, + _ => 90, }; - - format!( - r#"--- -title: "{}" -created: {} -tags: [{}]{} ---- - -{} -"#, - title.trim_start_matches('#').trim(), - now, - tags_str, - filenames_str, - content - ) + let review_after = (now + Duration::days(review_days)).format("%Y-%m-%d").to_string(); + + KnowledgeFrontmatter { + id: Some(Uuid::new_v4().to_string()), + title: title.map(|t| t.to_string()), + tags: tags.to_vec(), + created: Some(created.clone()), + updated: Some(created), + filenames: filenames.to_vec(), + links: links.to_vec(), + kind: Some(kind.to_string()), + status: Some("active".to_string()), + superseded_by: None, + deprecated_at: None, + review_after: Some(review_after), + } } async fn get_knowledge_dir(gcx: Arc>) -> Result { @@ -86,8 +94,7 @@ async fn get_knowledge_dir(gcx: Arc>) -> Result>, - tags: &[String], - filenames: &[String], + frontmatter: &KnowledgeFrontmatter, content: &str, ) -> Result { let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; @@ -100,7 +107,7 @@ pub async fn memories_add( return Err(format!("File already exists: {}", file_path.display())); } - let md_content = create_markdown_content(tags, filenames, content); + let md_content = format!("{}\n\n{}", frontmatter.to_yaml(), content); fs::write(&file_path, &md_content).await.map_err(|e| format!("Failed to write knowledge file: {}", e))?; info!("Created knowledge entry: {}", file_path.display()); @@ -112,13 +119,14 @@ pub async fn memories_add( Ok(file_path) } + + pub async fn memories_search( gcx: Arc>, query: &str, top_n: usize, ) -> Result, String> { let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; - let knowledge_prefix = knowledge_dir.to_string_lossy().to_string(); let has_vecdb = gcx.read().await.vec_db.lock().await.is_some(); if has_vecdb { @@ -140,20 +148,28 @@ pub async fn memories_search( Err(_) => continue, }; - let (frontmatter, _) = MarkdownFrontmatter::parse(&text); + let (frontmatter, _content_start) = KnowledgeFrontmatter::parse(&text); + + if frontmatter.is_archived() { + continue; + } + let lines: Vec<&str> = text.lines().collect(); let start = (rec.start_line as usize).min(lines.len().saturating_sub(1)); let end = (rec.end_line as usize).min(lines.len().saturating_sub(1)); let snippet = lines[start..=end].join("\n"); + let id = frontmatter.id.clone().unwrap_or_else(|| path_str.clone()); + records.push(MemoRecord { - memid: format!("{}:{}-{}", path_str, rec.start_line, rec.end_line), + memid: format!("{}:{}-{}", id, rec.start_line, rec.end_line), tags: frontmatter.tags, content: snippet, file_path: Some(rec.file_path.clone()), line_range: Some((rec.start_line, rec.end_line)), title: frontmatter.title, created: frontmatter.created, + kind: frontmatter.kind, }); if records.len() >= top_n { @@ -166,29 +182,31 @@ pub async fn memories_search( } } - memories_search_fallback(gcx, query, top_n, &knowledge_prefix).await + memories_search_fallback(gcx, query, top_n, &knowledge_dir).await } async fn memories_search_fallback( gcx: Arc>, query: &str, top_n: usize, - knowledge_dir: &str, + knowledge_dir: &PathBuf, ) -> Result, String> { let query_lower = query.to_lowercase(); let query_words: Vec<&str> = query_lower.split_whitespace().collect(); let mut scored_results: Vec<(usize, MemoRecord)> = Vec::new(); - let knowledge_path = PathBuf::from(knowledge_dir); - if !knowledge_path.exists() { + if !knowledge_dir.exists() { return Ok(vec![]); } - for entry in WalkDir::new(&knowledge_path).into_iter().filter_map(|e| e.ok()) { + for entry in WalkDir::new(knowledge_dir).into_iter().filter_map(|e| e.ok()) { let path = entry.path(); if !path.is_file() { continue; } + if path.to_string_lossy().contains("/archive/") { + continue; + } let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); if ext != "md" && ext != "mdx" { continue; @@ -205,15 +223,23 @@ async fn memories_search_fallback( continue; } - let (frontmatter, _) = MarkdownFrontmatter::parse(&text); + let (frontmatter, content_start) = KnowledgeFrontmatter::parse(&text); + if frontmatter.is_archived() { + continue; + } + + let id = frontmatter.id.clone().unwrap_or_else(|| path.to_string_lossy().to_string()); + let content_preview: String = text[content_start..].chars().take(500).collect(); + scored_results.push((score, MemoRecord { - memid: path.to_string_lossy().to_string(), + memid: id, tags: frontmatter.tags, - content: text.chars().take(500).collect(), + content: content_preview, file_path: Some(path.to_path_buf()), line_range: None, title: frontmatter.title, created: frontmatter.created, + kind: frontmatter.kind, })); } @@ -232,8 +258,15 @@ pub async fn save_trajectory( let filename = generate_filename(compressed_trajectory); let file_path = trajectories_dir.join(&filename); - let tags = vec!["trajectory".to_string()]; - let md_content = create_markdown_content(&tags, &[], compressed_trajectory); + let frontmatter = create_frontmatter( + compressed_trajectory.lines().next(), + &["trajectory".to_string()], + &[], + &[], + "trajectory", + ); + + let md_content = format!("{}\n\n{}", frontmatter.to_yaml(), compressed_trajectory); fs::write(&file_path, &md_content).await.map_err(|e| format!("Failed to write trajectory file: {}", e))?; info!("Saved trajectory: {}", file_path.display()); @@ -244,3 +277,50 @@ pub async fn save_trajectory( Ok(file_path) } + +pub async fn deprecate_document( + gcx: Arc>, + doc_path: &PathBuf, + superseded_by: Option<&str>, + reason: &str, +) -> Result<(), String> { + let text = get_file_text_from_memory_or_disk(gcx.clone(), doc_path).await + .map_err(|e| format!("Failed to read document: {}", e))?; + + let (mut frontmatter, content_start) = KnowledgeFrontmatter::parse(&text); + let content = &text[content_start..]; + + frontmatter.status = Some("deprecated".to_string()); + frontmatter.deprecated_at = Some(Local::now().format("%Y-%m-%d").to_string()); + if let Some(new_id) = superseded_by { + frontmatter.superseded_by = Some(new_id.to_string()); + } + + let deprecated_banner = format!("\n\n> ⚠️ **DEPRECATED**: {}\n", reason); + let new_content = format!("{}\n{}{}", frontmatter.to_yaml(), deprecated_banner, content); + + fs::write(doc_path, new_content).await.map_err(|e| format!("Failed to write: {}", e))?; + + info!("Deprecated document: {}", doc_path.display()); + + if let Some(vecdb) = gcx.read().await.vec_db.lock().await.as_ref() { + vecdb.vectorizer_enqueue_files(&vec![doc_path.to_string_lossy().to_string()], true).await; + } + + Ok(()) +} + +pub async fn archive_document(gcx: Arc>, doc_path: &PathBuf) -> Result { + let knowledge_dir = get_knowledge_dir(gcx.clone()).await?; + let archive_dir = knowledge_dir.join("archive"); + fs::create_dir_all(&archive_dir).await.map_err(|e| format!("Failed to create archive dir: {}", e))?; + + let filename = doc_path.file_name().ok_or("Invalid filename")?; + let archive_path = archive_dir.join(filename); + + fs::rename(doc_path, &archive_path).await.map_err(|e| format!("Failed to move to archive: {}", e))?; + + info!("Archived document: {} -> {}", doc_path.display(), archive_path.display()); + + Ok(archive_path) +} diff --git a/refact-agent/engine/src/tools/tool_create_knowledge.rs b/refact-agent/engine/src/tools/tool_create_knowledge.rs index 14c75f6a6..97014bc31 100644 --- a/refact-agent/engine/src/tools/tool_create_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_create_knowledge.rs @@ -1,18 +1,38 @@ use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; +use regex::Regex; use serde_json::Value; -use tracing::info; +use tracing::{info, warn}; use tokio::sync::Mutex as AMutex; use crate::at_commands::at_commands::AtCommandsContext; use crate::call_validation::{ChatMessage, ChatContent, ContextEnum}; use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType}; +use crate::knowledge_graph::kg_subchat::{enrich_knowledge_metadata, check_deprecation}; +use crate::knowledge_graph::kg_structs::KnowledgeFrontmatter; +use crate::memories::{memories_add, deprecate_document}; +use crate::global_context::try_load_caps_quickly_if_not_present; pub struct ToolCreateKnowledge { pub config_path: String, } +fn extract_entities(content: &str) -> Vec { + let backtick_re = Regex::new(r"`([a-zA-Z_][a-zA-Z0-9_:]*(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)`").unwrap(); + backtick_re.captures_iter(content) + .map(|c| c.get(1).unwrap().as_str().to_string()) + .filter(|e| e.len() >= 3 && e.len() <= 100) + .collect() +} + +fn extract_file_paths(content: &str) -> Vec { + let path_re = Regex::new(r"(?:^|[\s`])((?:[a-zA-Z0-9_-]+/)+[a-zA-Z0-9_-]+\.[a-zA-Z0-9]+)").unwrap(); + path_re.captures_iter(content) + .map(|c| c.get(1).unwrap().as_str().to_string()) + .collect() +} + #[async_trait] impl Tool for ToolCreateKnowledge { fn as_any(&self) -> &dyn std::any::Any { self } @@ -27,22 +47,22 @@ impl Tool for ToolCreateKnowledge { }, agentic: true, experimental: false, - description: "Creates a new knowledge entry as a markdown file in the project's .refact_knowledge folder.".to_string(), + description: "Creates a new knowledge entry. Uses AI to enrich metadata and check for outdated documents.".to_string(), parameters: vec![ ToolParam { - name: "tags".to_string(), + name: "content".to_string(), param_type: "string".to_string(), - description: "Comma-separated tags for categorizing the knowledge entry, e.g. \"architecture, patterns, rust\"".to_string(), + description: "The knowledge content to store.".to_string(), }, ToolParam { - name: "filenames".to_string(), + name: "tags".to_string(), param_type: "string".to_string(), - description: "Comma-separated list of related file paths that this knowledge entry documents or references.".to_string(), + description: "Comma-separated tags (optional, will be auto-enriched).".to_string(), }, ToolParam { - name: "content".to_string(), + name: "filenames".to_string(), param_type: "string".to_string(), - description: "The knowledge content to store. Include comprehensive information about implementation details, code patterns, architectural decisions, or solutions.".to_string(), + description: "Comma-separated related file paths (optional, will be auto-enriched).".to_string(), }, ], parameters_required: vec!["content".to_string()], @@ -65,31 +85,163 @@ impl Tool for ToolCreateKnowledge { None => return Err("argument `content` is missing".to_string()), }; - let tags: Vec = match args.get("tags") { - Some(Value::String(s)) => s.split(',') - .map(|t| t.trim().to_string()) - .filter(|t| !t.is_empty()) - .collect(), - Some(Value::Array(arr)) => arr.iter() - .filter_map(|v| v.as_str().map(String::from)) - .collect(), - _ => vec!["knowledge".to_string()], + let user_tags: Vec = match args.get("tags") { + Some(Value::String(s)) => s.split(',').map(|t| t.trim().to_string()).filter(|t| !t.is_empty()).collect(), + _ => vec![], }; - let tags = if tags.is_empty() { vec!["knowledge".to_string()] } else { tags }; - let filenames: Vec = match args.get("filenames") { - Some(Value::String(s)) => s.split(',') - .map(|f| f.trim().to_string()) - .filter(|f| !f.is_empty()) - .collect(), + let user_filenames: Vec = match args.get("filenames") { + Some(Value::String(s)) => s.split(',').map(|f| f.trim().to_string()).filter(|f| !f.is_empty()).collect(), _ => vec![], }; - let file_path = crate::memories::memories_add(gcx, &tags, &filenames, &content).await?; + let entities = extract_entities(&content); + let detected_paths = extract_file_paths(&content); + + let caps = try_load_caps_quickly_if_not_present(gcx.clone(), 0).await + .map_err(|e| format!("Failed to load caps: {}", e.message))?; + let light_model = if caps.defaults.chat_light_model.is_empty() { + caps.defaults.chat_default_model.clone() + } else { + caps.defaults.chat_light_model.clone() + }; + + let kg = crate::knowledge_graph::build_knowledge_graph(gcx.clone()).await; + + let candidate_files: Vec = { + let mut files = user_filenames.clone(); + files.extend(detected_paths); + files.into_iter().take(30).collect() + }; + + let candidate_docs: Vec<(String, String)> = kg.active_docs() + .take(20) + .map(|d| { + let id = d.frontmatter.id.clone().unwrap_or_else(|| d.path.to_string_lossy().to_string()); + let title = d.frontmatter.title.clone().unwrap_or_else(|| "Untitled".to_string()); + (id, title) + }) + .collect(); + + let enrichment = enrich_knowledge_metadata( + ccx.clone(), + &light_model, + &content, + &entities, + &candidate_files, + &candidate_docs, + ).await; + + let (final_title, final_tags, final_filenames, final_kind, final_links, review_days) = match enrichment { + Ok(e) => { + let mut tags = user_tags.clone(); + tags.extend(e.tags); + tags.sort(); + tags.dedup(); + + let mut files = user_filenames.clone(); + files.extend(e.filenames); + files.sort(); + files.dedup(); + + let kind = e.kind.unwrap_or_else(|| if files.is_empty() { "domain".to_string() } else { "code".to_string() }); + + ( + e.title.or_else(|| content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())), + if tags.is_empty() { vec!["knowledge".to_string()] } else { tags }, + files, + kind, + e.links, + e.review_after_days.unwrap_or(90), + ) + } + Err(e) => { + warn!("Enrichment failed, using defaults: {}", e); + let tags = if user_tags.is_empty() { vec!["knowledge".to_string()] } else { user_tags }; + let kind = if user_filenames.is_empty() { "domain".to_string() } else { "code".to_string() }; + ( + content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string()), + tags, + user_filenames, + kind, + vec![], + 90, + ) + } + }; + + let now = chrono::Local::now(); + let frontmatter = KnowledgeFrontmatter { + id: Some(uuid::Uuid::new_v4().to_string()), + title: final_title.clone(), + tags: final_tags.clone(), + created: Some(now.format("%Y-%m-%d").to_string()), + updated: Some(now.format("%Y-%m-%d").to_string()), + filenames: final_filenames.clone(), + links: final_links, + kind: Some(final_kind.clone()), + status: Some("active".to_string()), + superseded_by: None, + deprecated_at: None, + review_after: Some((now + chrono::Duration::days(review_days)).format("%Y-%m-%d").to_string()), + }; + + let file_path = memories_add(gcx.clone(), &frontmatter, &content).await?; + let new_doc_id = frontmatter.id.clone().unwrap(); + + let deprecation_candidates = kg.get_deprecation_candidates( + &final_tags, + &final_filenames, + &entities, + Some(&new_doc_id), + ); + + let mut deprecated_count = 0; + if !deprecation_candidates.is_empty() { + let snippet: String = content.chars().take(500).collect(); + + match check_deprecation( + ccx.clone(), + &light_model, + final_title.as_deref().unwrap_or("Untitled"), + &final_tags, + &final_filenames, + &snippet, + &deprecation_candidates, + ).await { + Ok(result) => { + for decision in result.deprecate { + if decision.confidence >= 0.75 { + if let Some(doc) = kg.get_doc_by_id(&decision.target_id) { + if let Err(e) = deprecate_document( + gcx.clone(), + &doc.path, + Some(&new_doc_id), + &decision.reason, + ).await { + warn!("Failed to deprecate {}: {}", decision.target_id, e); + } else { + deprecated_count += 1; + info!("Deprecated {} (confidence: {:.2}): {}", decision.target_id, decision.confidence, decision.reason); + } + } + } + } + } + Err(e) => { + warn!("Deprecation check failed: {}", e); + } + } + } + + let mut result_msg = format!("Knowledge entry created: {}", file_path.display()); + if deprecated_count > 0 { + result_msg.push_str(&format!("\n{} outdated document(s) marked as deprecated.", deprecated_count)); + } Ok((false, vec![ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), - content: ChatContent::SimpleText(format!("Knowledge entry created: {}", file_path.display())), + content: ChatContent::SimpleText(result_msg), tool_calls: None, tool_call_id: tool_call_id.clone(), ..Default::default() @@ -97,6 +249,6 @@ impl Tool for ToolCreateKnowledge { } fn tool_depends_on(&self) -> Vec { - vec![] + vec!["knowledge".to_string()] } } diff --git a/refact-agent/engine/src/tools/tool_knowledge.rs b/refact-agent/engine/src/tools/tool_knowledge.rs index 48e062551..badeb0f06 100644 --- a/refact-agent/engine/src/tools/tool_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_knowledge.rs @@ -1,14 +1,16 @@ use std::sync::Arc; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use serde_json::Value; use tracing::info; use tokio::sync::Mutex as AMutex; use async_trait::async_trait; +use std::collections::HashMap; use crate::at_commands::at_commands::AtCommandsContext; use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType}; use crate::call_validation::{ChatMessage, ChatContent, ContextEnum}; use crate::memories::memories_search; +use crate::knowledge_graph::build_knowledge_graph; pub struct ToolGetKnowledge { pub config_path: String, @@ -28,12 +30,12 @@ impl Tool for ToolGetKnowledge { }, agentic: true, experimental: false, - description: "Searches project knowledge base for relevant information. Use this to find existing documentation, patterns, decisions, and solutions.".to_string(), + description: "Searches project knowledge base for relevant information. Uses semantic search and knowledge graph expansion.".to_string(), parameters: vec![ ToolParam { name: "search_key".to_string(), param_type: "string".to_string(), - description: "Search query for the knowledge database. Describe what you're looking for.".to_string(), + description: "Search query for the knowledge database.".to_string(), } ], parameters_required: vec!["search_key".to_string()], @@ -56,17 +58,54 @@ impl Tool for ToolGetKnowledge { None => return Err("argument `search_key` is missing".to_string()), }; - let memories = memories_search(gcx, &search_key, 5).await?; + let memories = memories_search(gcx.clone(), &search_key, 5).await?; let mut seen_memids = HashSet::new(); - let unique_memories: Vec<_> = memories.into_iter() + let mut unique_memories: Vec<_> = memories.into_iter() .filter(|m| seen_memids.insert(m.memid.clone())) .collect(); + if !unique_memories.is_empty() { + let kg = build_knowledge_graph(gcx.clone()).await; + + let initial_ids: Vec = unique_memories.iter() + .filter_map(|m| m.file_path.as_ref()) + .filter_map(|p| kg.get_doc_by_path(p)) + .filter_map(|d| d.frontmatter.id.clone()) + .collect(); + + let expanded_ids = kg.expand_search_results(&initial_ids, 3); + + for id in expanded_ids { + if let Some(doc) = kg.get_doc_by_id(&id) { + if doc.frontmatter.is_active() && !seen_memids.contains(&id) { + seen_memids.insert(id.clone()); + let snippet: String = doc.content.chars().take(500).collect(); + unique_memories.push(crate::memories::MemoRecord { + memid: id, + tags: doc.frontmatter.tags.clone(), + content: snippet, + file_path: Some(doc.path.clone()), + line_range: None, + title: doc.frontmatter.title.clone(), + created: doc.frontmatter.created.clone(), + kind: doc.frontmatter.kind.clone(), + }); + } + } + } + } + + unique_memories.sort_by(|a, b| { + let a_is_traj = a.kind.as_deref() == Some("trajectory"); + let b_is_traj = b.kind.as_deref() == Some("trajectory"); + a_is_traj.cmp(&b_is_traj) + }); + let memories_str = if unique_memories.is_empty() { "No relevant knowledge found.".to_string() } else { - unique_memories.iter().map(|m| { + unique_memories.iter().take(8).map(|m| { let mut result = String::new(); if let Some(path) = &m.file_path { result.push_str(&format!("📄 {}", path.display())); @@ -78,11 +117,14 @@ impl Tool for ToolGetKnowledge { if let Some(title) = &m.title { result.push_str(&format!("📌 {}\n", title)); } + if let Some(kind) = &m.kind { + result.push_str(&format!("📦 {}\n", kind)); + } if !m.tags.is_empty() { result.push_str(&format!("🏷️ {}\n", m.tags.join(", "))); } result.push_str(&m.content); - result.push_str("\n\n"); + result.push_str("\n\n---\n"); result }).collect() }; @@ -97,6 +139,6 @@ impl Tool for ToolGetKnowledge { } fn tool_depends_on(&self) -> Vec { - vec![] + vec!["knowledge".to_string()] } } diff --git a/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs b/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs index db7337908..25142acc7 100644 --- a/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs +++ b/refact-agent/engine/src/vecdb/vdb_markdown_splitter.rs @@ -7,57 +7,9 @@ use crate::files_in_workspace::Document; use crate::global_context::GlobalContext; use crate::vecdb::vdb_structs::SplitResult; use crate::ast::chunk_utils::official_text_hashing_function; +use crate::knowledge_graph::KnowledgeFrontmatter; -#[derive(Debug, Clone, Default)] -pub struct MarkdownFrontmatter { - pub title: Option, - pub tags: Vec, - pub created: Option, - pub updated: Option, -} - -impl MarkdownFrontmatter { - pub fn parse(content: &str) -> (Self, usize) { - let mut frontmatter = Self::default(); - let mut end_offset = 0; - - if content.starts_with("---") { - if let Some(end_idx) = content[3..].find("\n---") { - let yaml_content = &content[3..3 + end_idx]; - end_offset = 3 + end_idx + 4; - if content.len() > end_offset && content.as_bytes()[end_offset] == b'\n' { - end_offset += 1; - } - - for line in yaml_content.lines() { - let line = line.trim(); - if let Some(pos) = line.find(':') { - let key = line[..pos].trim(); - let value = line[pos + 1..].trim().trim_matches('"').trim_matches('\''); - match key { - "title" => frontmatter.title = Some(value.to_string()), - "created" => frontmatter.created = Some(value.to_string()), - "updated" => frontmatter.updated = Some(value.to_string()), - "tags" => { - if value.starts_with('[') && value.ends_with(']') { - frontmatter.tags = value[1..value.len() - 1] - .split(',') - .map(|s| s.trim().trim_matches('"').trim_matches('\'').to_string()) - .filter(|s| !s.is_empty()) - .collect(); - } else if !value.is_empty() { - frontmatter.tags = vec![value.to_string()]; - } - } - _ => {} - } - } - } - } - } - (frontmatter, end_offset) - } -} +pub use crate::knowledge_graph::KnowledgeFrontmatter as MarkdownFrontmatter; #[derive(Debug, Clone)] struct MarkdownSection { @@ -111,7 +63,7 @@ impl MarkdownFileSplitter { Ok(results) } - fn format_frontmatter_chunk(&self, fm: &MarkdownFrontmatter) -> String { + fn format_frontmatter_chunk(&self, fm: &KnowledgeFrontmatter) -> String { let mut parts = Vec::new(); if let Some(title) = &fm.title { parts.push(format!("Title: {}", title)); @@ -119,6 +71,12 @@ impl MarkdownFileSplitter { if !fm.tags.is_empty() { parts.push(format!("Tags: {}", fm.tags.join(", "))); } + if let Some(kind) = &fm.kind { + parts.push(format!("Kind: {}", kind)); + } + if !fm.filenames.is_empty() { + parts.push(format!("Files: {}", fm.filenames.join(", "))); + } parts.join("\n") } @@ -250,21 +208,25 @@ mod tests { title: "Test Document" created: 2024-12-17 tags: ["rust", "testing"] +filenames: ["src/main.rs"] +kind: code --- # Hello World "#; - let (fm, offset) = MarkdownFrontmatter::parse(content); + let (fm, offset) = KnowledgeFrontmatter::parse(content); assert_eq!(fm.title, Some("Test Document".to_string())); assert_eq!(fm.tags, vec!["rust", "testing"]); assert_eq!(fm.created, Some("2024-12-17".to_string())); + assert_eq!(fm.filenames, vec!["src/main.rs"]); + assert_eq!(fm.kind, Some("code".to_string())); assert!(offset > 0); } #[test] fn test_frontmatter_no_frontmatter() { let content = "# Just a heading\n\nSome content"; - let (fm, offset) = MarkdownFrontmatter::parse(content); + let (fm, offset) = KnowledgeFrontmatter::parse(content); assert!(fm.title.is_none()); assert!(fm.tags.is_empty()); assert_eq!(offset, 0); From 319d14f12b4b802adaf65b934281849498546f53 Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 16:28:40 +1030 Subject: [PATCH 6/8] feat: add /v1/knowledge-graph endpoint and stats logging - New HTTP endpoint GET /v1/knowledge-graph returns JSON: - nodes: [{id, node_type, label}] for docs, tags, files, entities - edges: [{source, target, edge_type}] for all relationships - stats: doc_count, tag_count, file_count, entity_count, edge_count, etc. - Enhanced stats logging when knowledge graph is built: - Total docs (active, deprecated, trajectories, code) - Tags, files, entities counts - Graph edge count New file: src/http/routers/v1/knowledge_graph.rs --- refact-agent/engine/src/http/routers/v1.rs | 3 + .../src/http/routers/v1/knowledge_graph.rs | 172 ++++++++++++++++++ .../engine/src/knowledge_graph/kg_builder.rs | 22 ++- 3 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 refact-agent/engine/src/http/routers/v1/knowledge_graph.rs diff --git a/refact-agent/engine/src/http/routers/v1.rs b/refact-agent/engine/src/http/routers/v1.rs index 60669b16e..18ea86c60 100644 --- a/refact-agent/engine/src/http/routers/v1.rs +++ b/refact-agent/engine/src/http/routers/v1.rs @@ -35,6 +35,7 @@ use crate::http::routers::v1::providers::{handle_v1_providers, handle_v1_provide handle_v1_delete_model, handle_v1_delete_provider, handle_v1_model_default, handle_v1_completion_model_families}; use crate::http::routers::v1::vecdb::{handle_v1_vecdb_search, handle_v1_vecdb_status}; +use crate::http::routers::v1::knowledge_graph::handle_v1_knowledge_graph; use crate::http::routers::v1::v1_integrations::{handle_v1_integration_get, handle_v1_integration_icon, handle_v1_integration_save, handle_v1_integration_delete, handle_v1_integrations, handle_v1_integrations_filtered, handle_v1_integrations_mcp_logs}; use crate::http::routers::v1::file_edit_tools::handle_v1_file_edit_tool_dry_run; use crate::http::routers::v1::code_edit::handle_v1_code_edit; @@ -69,6 +70,7 @@ mod code_edit; mod v1_integrations; pub mod vecdb; mod workspace; +mod knowledge_graph; pub fn make_v1_router() -> Router { let builder = Router::new() @@ -168,6 +170,7 @@ pub fn make_v1_router() -> Router { let builder = builder .route("/vdb-search", post(handle_v1_vecdb_search)) .route("/vdb-status", get(handle_v1_vecdb_status)) + .route("/knowledge-graph", get(handle_v1_knowledge_graph)) .route("/trajectory-save", post(handle_v1_trajectory_save)) .route("/trajectory-compress", post(handle_v1_trajectory_compress)) ; diff --git a/refact-agent/engine/src/http/routers/v1/knowledge_graph.rs b/refact-agent/engine/src/http/routers/v1/knowledge_graph.rs new file mode 100644 index 000000000..6fecce56f --- /dev/null +++ b/refact-agent/engine/src/http/routers/v1/knowledge_graph.rs @@ -0,0 +1,172 @@ +use axum::Extension; +use axum::response::Result; +use hyper::{Body, Response, StatusCode}; +use serde::Serialize; + +use crate::custom_error::ScratchError; +use crate::global_context::SharedGlobalContext; +use crate::knowledge_graph::build_knowledge_graph; + +#[derive(Serialize)] +struct KgNodeJson { + id: String, + node_type: String, + label: String, +} + +#[derive(Serialize)] +struct KgEdgeJson { + source: String, + target: String, + edge_type: String, +} + +#[derive(Serialize)] +struct KgStatsJson { + doc_count: usize, + tag_count: usize, + file_count: usize, + entity_count: usize, + edge_count: usize, + active_docs: usize, + deprecated_docs: usize, + trajectory_count: usize, +} + +#[derive(Serialize)] +struct KnowledgeGraphJson { + nodes: Vec, + edges: Vec, + stats: KgStatsJson, +} + +pub async fn handle_v1_knowledge_graph( + Extension(gcx): Extension, +) -> Result, ScratchError> { + let kg = build_knowledge_graph(gcx).await; + + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + + for (id, doc) in &kg.docs { + let label = doc.frontmatter.title.clone().unwrap_or_else(|| { + doc.path.file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| id.clone()) + }); + let node_type = match doc.frontmatter.status.as_deref() { + Some("deprecated") => "doc_deprecated", + Some("archived") => "doc_archived", + _ => match doc.frontmatter.kind.as_deref() { + Some("trajectory") => "doc_trajectory", + Some("code") => "doc_code", + Some("decision") => "doc_decision", + _ => "doc", + } + }; + nodes.push(KgNodeJson { + id: id.clone(), + node_type: node_type.to_string(), + label, + }); + } + + for (tag, _) in &kg.tag_index { + nodes.push(KgNodeJson { + id: format!("tag:{}", tag), + node_type: "tag".to_string(), + label: tag.clone(), + }); + } + + for (file, _) in &kg.file_index { + let label = std::path::Path::new(file) + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| file.clone()); + nodes.push(KgNodeJson { + id: format!("file:{}", file), + node_type: "file".to_string(), + label, + }); + } + + for (entity, _) in &kg.entity_index { + nodes.push(KgNodeJson { + id: format!("entity:{}", entity), + node_type: "entity".to_string(), + label: entity.clone(), + }); + } + + for (id, doc) in &kg.docs { + for tag in &doc.frontmatter.tags { + edges.push(KgEdgeJson { + source: id.clone(), + target: format!("tag:{}", tag.to_lowercase()), + edge_type: "tagged_with".to_string(), + }); + } + for file in &doc.frontmatter.filenames { + edges.push(KgEdgeJson { + source: id.clone(), + target: format!("file:{}", file), + edge_type: "references_file".to_string(), + }); + } + for entity in &doc.entities { + edges.push(KgEdgeJson { + source: id.clone(), + target: format!("entity:{}", entity), + edge_type: "mentions".to_string(), + }); + } + for link in &doc.frontmatter.links { + if kg.docs.contains_key(link) { + edges.push(KgEdgeJson { + source: id.clone(), + target: link.clone(), + edge_type: "links_to".to_string(), + }); + } + } + if let Some(superseded_by) = &doc.frontmatter.superseded_by { + if kg.docs.contains_key(superseded_by) { + edges.push(KgEdgeJson { + source: id.clone(), + target: superseded_by.clone(), + edge_type: "superseded_by".to_string(), + }); + } + } + } + + let active_docs = kg.docs.values().filter(|d| d.frontmatter.is_active()).count(); + let deprecated_docs = kg.docs.values().filter(|d| d.frontmatter.is_deprecated()).count(); + let trajectory_count = kg.docs.values() + .filter(|d| d.frontmatter.kind.as_deref() == Some("trajectory")) + .count(); + + let stats = KgStatsJson { + doc_count: kg.docs.len(), + tag_count: kg.tag_index.len(), + file_count: kg.file_index.len(), + entity_count: kg.entity_index.len(), + edge_count: edges.len(), + active_docs, + deprecated_docs, + trajectory_count, + }; + + let response = KnowledgeGraphJson { nodes, edges, stats }; + + let json_string = serde_json::to_string_pretty(&response).map_err(|e| { + ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("JSON serialization error: {}", e)) + })?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .body(Body::from(json_string)) + .unwrap()) +} diff --git a/refact-agent/engine/src/knowledge_graph/kg_builder.rs b/refact-agent/engine/src/knowledge_graph/kg_builder.rs index c5c60a302..2b6b8758b 100644 --- a/refact-agent/engine/src/knowledge_graph/kg_builder.rs +++ b/refact-agent/engine/src/knowledge_graph/kg_builder.rs @@ -98,12 +98,22 @@ pub async fn build_knowledge_graph(gcx: Arc>) -> Knowledg } graph.link_docs(); - info!("knowledge_graph: built with {} documents, {} tags, {} file refs, {} entities", - doc_count, - graph.tag_index.len(), - graph.file_index.len(), - graph.entity_index.len() - ); + + let active_count = graph.docs.values().filter(|d| d.frontmatter.is_active()).count(); + let deprecated_count = graph.docs.values().filter(|d| d.frontmatter.is_deprecated()).count(); + let trajectory_count = graph.docs.values() + .filter(|d| d.frontmatter.kind.as_deref() == Some("trajectory")) + .count(); + let code_count = graph.docs.values() + .filter(|d| d.frontmatter.kind.as_deref() == Some("code")) + .count(); + + info!("knowledge_graph: built successfully"); + info!(" Documents: {} total ({} active, {} deprecated, {} trajectories, {} code)", + doc_count, active_count, deprecated_count, trajectory_count, code_count); + info!(" Tags: {}, Files: {}, Entities: {}", + graph.tag_index.len(), graph.file_index.len(), graph.entity_index.len()); + info!(" Graph edges: {}", graph.graph.edge_count()); graph } From ae05c35e5d8561ac22b93893b98b7ab2e4c94d33 Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 16:49:52 +1030 Subject: [PATCH 7/8] Add memories_add_enriched for AI-powered knowledge creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add EnrichmentParams struct and memories_add_enriched() to memories.rs - Refactor tool_create_knowledge.rs to use memories_add_enriched (254→93 lines) - Add enriched memory creation to tool_deep_research.rs - Add enriched memory creation to tool_strategic_planning.rs - All knowledge entries now get AI-enriched metadata and auto-deprecation --- refact-agent/engine/src/memories.rs | 177 +++++++++++++++++- .../engine/src/tools/tool_create_knowledge.rs | 172 +---------------- .../engine/src/tools/tool_deep_research.rs | 18 ++ .../src/tools/tool_strategic_planning.rs | 17 ++ 4 files changed, 219 insertions(+), 165 deletions(-) diff --git a/refact-agent/engine/src/memories.rs b/refact-agent/engine/src/memories.rs index c2c888d7b..c13b9bc4c 100644 --- a/refact-agent/engine/src/memories.rs +++ b/refact-agent/engine/src/memories.rs @@ -1,18 +1,23 @@ use std::path::PathBuf; use std::sync::Arc; use chrono::{Local, Duration}; +use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock as ARwLock; +use tokio::sync::Mutex as AMutex; use tokio::fs; -use tracing::info; +use tracing::{info, warn}; use uuid::Uuid; use walkdir::WalkDir; +use crate::at_commands::at_commands::AtCommandsContext; use crate::file_filter::KNOWLEDGE_FOLDER_NAME; use crate::files_correction::get_project_dirs; use crate::files_in_workspace::get_file_text_from_memory_or_disk; -use crate::global_context::GlobalContext; +use crate::global_context::{GlobalContext, try_load_caps_quickly_if_not_present}; use crate::knowledge_graph::kg_structs::KnowledgeFrontmatter; +use crate::knowledge_graph::kg_subchat::{enrich_knowledge_metadata, check_deprecation}; +use crate::knowledge_graph::build_knowledge_graph; use crate::vecdb::vdb_structs::VecdbSearch; #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq)] @@ -324,3 +329,171 @@ pub async fn archive_document(gcx: Arc>, doc_path: &PathB Ok(archive_path) } + +fn extract_entities(content: &str) -> Vec { + let backtick_re = Regex::new(r"`([a-zA-Z_][a-zA-Z0-9_:]*(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)`").unwrap(); + backtick_re.captures_iter(content) + .map(|c| c.get(1).unwrap().as_str().to_string()) + .filter(|e| e.len() >= 3 && e.len() <= 100) + .collect() +} + +fn extract_file_paths(content: &str) -> Vec { + let path_re = Regex::new(r"(?:^|[\s`])((?:[a-zA-Z0-9_-]+/)+[a-zA-Z0-9_-]+\.[a-zA-Z0-9]+)").unwrap(); + path_re.captures_iter(content) + .map(|c| c.get(1).unwrap().as_str().to_string()) + .collect() +} + +pub struct EnrichmentParams { + pub base_tags: Vec, + pub base_filenames: Vec, + pub base_kind: String, + pub base_title: Option, +} + +pub async fn memories_add_enriched( + ccx: Arc>, + content: &str, + params: EnrichmentParams, +) -> Result { + let gcx = ccx.lock().await.global_context.clone(); + + let entities = extract_entities(content); + let detected_paths = extract_file_paths(content); + + let caps = try_load_caps_quickly_if_not_present(gcx.clone(), 0).await + .map_err(|e| format!("Failed to load caps: {}", e.message))?; + let light_model = if caps.defaults.chat_light_model.is_empty() { + caps.defaults.chat_default_model.clone() + } else { + caps.defaults.chat_light_model.clone() + }; + + let kg = build_knowledge_graph(gcx.clone()).await; + + let candidate_files: Vec = { + let mut files = params.base_filenames.clone(); + files.extend(detected_paths); + files.into_iter().take(30).collect() + }; + + let candidate_docs: Vec<(String, String)> = kg.active_docs() + .take(20) + .map(|d| { + let id = d.frontmatter.id.clone().unwrap_or_else(|| d.path.to_string_lossy().to_string()); + let title = d.frontmatter.title.clone().unwrap_or_else(|| "Untitled".to_string()); + (id, title) + }) + .collect(); + + let enrichment = enrich_knowledge_metadata( + ccx.clone(), + &light_model, + content, + &entities, + &candidate_files, + &candidate_docs, + ).await; + + let (final_title, final_tags, final_filenames, final_kind, final_links, review_days) = match enrichment { + Ok(e) => { + let mut tags = params.base_tags.clone(); + tags.extend(e.tags); + tags.sort(); + tags.dedup(); + + let mut files = params.base_filenames.clone(); + files.extend(e.filenames); + files.sort(); + files.dedup(); + + let kind = e.kind.unwrap_or_else(|| params.base_kind.clone()); + + ( + e.title.or(params.base_title.clone()).or_else(|| content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())), + if tags.is_empty() { vec![params.base_kind.clone()] } else { tags }, + files, + kind, + e.links, + e.review_after_days.unwrap_or(90), + ) + } + Err(e) => { + warn!("Enrichment failed, using defaults: {}", e); + let tags = if params.base_tags.is_empty() { vec![params.base_kind.clone()] } else { params.base_tags }; + ( + params.base_title.or_else(|| content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())), + tags, + params.base_filenames, + params.base_kind, + vec![], + 90, + ) + } + }; + + let now = Local::now(); + let frontmatter = KnowledgeFrontmatter { + id: Some(Uuid::new_v4().to_string()), + title: final_title.clone(), + tags: final_tags.clone(), + created: Some(now.format("%Y-%m-%d").to_string()), + updated: Some(now.format("%Y-%m-%d").to_string()), + filenames: final_filenames.clone(), + links: final_links, + kind: Some(final_kind), + status: Some("active".to_string()), + superseded_by: None, + deprecated_at: None, + review_after: Some((now + Duration::days(review_days)).format("%Y-%m-%d").to_string()), + }; + + let file_path = memories_add(gcx.clone(), &frontmatter, content).await?; + let new_doc_id = frontmatter.id.clone().unwrap(); + + let deprecation_candidates = kg.get_deprecation_candidates( + &final_tags, + &final_filenames, + &entities, + Some(&new_doc_id), + ); + + if !deprecation_candidates.is_empty() { + let snippet: String = content.chars().take(500).collect(); + + match check_deprecation( + ccx.clone(), + &light_model, + final_title.as_deref().unwrap_or("Untitled"), + &final_tags, + &final_filenames, + &snippet, + &deprecation_candidates, + ).await { + Ok(result) => { + for decision in result.deprecate { + if decision.confidence >= 0.75 { + if let Some(doc) = kg.get_doc_by_id(&decision.target_id) { + if let Err(e) = deprecate_document( + gcx.clone(), + &doc.path, + Some(&new_doc_id), + &decision.reason, + ).await { + warn!("Failed to deprecate {}: {}", decision.target_id, e); + } else { + info!("Deprecated {} (confidence: {:.2}): {}", decision.target_id, decision.confidence, decision.reason); + } + } + } + } + } + Err(e) => { + warn!("Deprecation check failed: {}", e); + } + } + } + + Ok(file_path) +} diff --git a/refact-agent/engine/src/tools/tool_create_knowledge.rs b/refact-agent/engine/src/tools/tool_create_knowledge.rs index 97014bc31..ad4085f0b 100644 --- a/refact-agent/engine/src/tools/tool_create_knowledge.rs +++ b/refact-agent/engine/src/tools/tool_create_knowledge.rs @@ -1,38 +1,19 @@ use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; -use regex::Regex; use serde_json::Value; -use tracing::{info, warn}; +use tracing::info; use tokio::sync::Mutex as AMutex; use crate::at_commands::at_commands::AtCommandsContext; use crate::call_validation::{ChatMessage, ChatContent, ContextEnum}; use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType}; -use crate::knowledge_graph::kg_subchat::{enrich_knowledge_metadata, check_deprecation}; -use crate::knowledge_graph::kg_structs::KnowledgeFrontmatter; -use crate::memories::{memories_add, deprecate_document}; -use crate::global_context::try_load_caps_quickly_if_not_present; +use crate::memories::{memories_add_enriched, EnrichmentParams}; pub struct ToolCreateKnowledge { pub config_path: String, } -fn extract_entities(content: &str) -> Vec { - let backtick_re = Regex::new(r"`([a-zA-Z_][a-zA-Z0-9_:]*(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)`").unwrap(); - backtick_re.captures_iter(content) - .map(|c| c.get(1).unwrap().as_str().to_string()) - .filter(|e| e.len() >= 3 && e.len() <= 100) - .collect() -} - -fn extract_file_paths(content: &str) -> Vec { - let path_re = Regex::new(r"(?:^|[\s`])((?:[a-zA-Z0-9_-]+/)+[a-zA-Z0-9_-]+\.[a-zA-Z0-9]+)").unwrap(); - path_re.captures_iter(content) - .map(|c| c.get(1).unwrap().as_str().to_string()) - .collect() -} - #[async_trait] impl Tool for ToolCreateKnowledge { fn as_any(&self) -> &dyn std::any::Any { self } @@ -77,8 +58,6 @@ impl Tool for ToolCreateKnowledge { ) -> Result<(bool, Vec), String> { info!("create_knowledge {:?}", args); - let gcx = ccx.lock().await.global_context.clone(); - let content = match args.get("content") { Some(Value::String(s)) => s.clone(), Some(v) => return Err(format!("argument `content` is not a string: {:?}", v)), @@ -95,149 +74,16 @@ impl Tool for ToolCreateKnowledge { _ => vec![], }; - let entities = extract_entities(&content); - let detected_paths = extract_file_paths(&content); - - let caps = try_load_caps_quickly_if_not_present(gcx.clone(), 0).await - .map_err(|e| format!("Failed to load caps: {}", e.message))?; - let light_model = if caps.defaults.chat_light_model.is_empty() { - caps.defaults.chat_default_model.clone() - } else { - caps.defaults.chat_light_model.clone() - }; - - let kg = crate::knowledge_graph::build_knowledge_graph(gcx.clone()).await; - - let candidate_files: Vec = { - let mut files = user_filenames.clone(); - files.extend(detected_paths); - files.into_iter().take(30).collect() + let enrichment_params = EnrichmentParams { + base_tags: user_tags, + base_filenames: user_filenames, + base_kind: "knowledge".to_string(), + base_title: None, }; - let candidate_docs: Vec<(String, String)> = kg.active_docs() - .take(20) - .map(|d| { - let id = d.frontmatter.id.clone().unwrap_or_else(|| d.path.to_string_lossy().to_string()); - let title = d.frontmatter.title.clone().unwrap_or_else(|| "Untitled".to_string()); - (id, title) - }) - .collect(); - - let enrichment = enrich_knowledge_metadata( - ccx.clone(), - &light_model, - &content, - &entities, - &candidate_files, - &candidate_docs, - ).await; - - let (final_title, final_tags, final_filenames, final_kind, final_links, review_days) = match enrichment { - Ok(e) => { - let mut tags = user_tags.clone(); - tags.extend(e.tags); - tags.sort(); - tags.dedup(); - - let mut files = user_filenames.clone(); - files.extend(e.filenames); - files.sort(); - files.dedup(); + let file_path = memories_add_enriched(ccx.clone(), &content, enrichment_params).await?; - let kind = e.kind.unwrap_or_else(|| if files.is_empty() { "domain".to_string() } else { "code".to_string() }); - - ( - e.title.or_else(|| content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())), - if tags.is_empty() { vec!["knowledge".to_string()] } else { tags }, - files, - kind, - e.links, - e.review_after_days.unwrap_or(90), - ) - } - Err(e) => { - warn!("Enrichment failed, using defaults: {}", e); - let tags = if user_tags.is_empty() { vec!["knowledge".to_string()] } else { user_tags }; - let kind = if user_filenames.is_empty() { "domain".to_string() } else { "code".to_string() }; - ( - content.lines().next().map(|l| l.trim_start_matches('#').trim().to_string()), - tags, - user_filenames, - kind, - vec![], - 90, - ) - } - }; - - let now = chrono::Local::now(); - let frontmatter = KnowledgeFrontmatter { - id: Some(uuid::Uuid::new_v4().to_string()), - title: final_title.clone(), - tags: final_tags.clone(), - created: Some(now.format("%Y-%m-%d").to_string()), - updated: Some(now.format("%Y-%m-%d").to_string()), - filenames: final_filenames.clone(), - links: final_links, - kind: Some(final_kind.clone()), - status: Some("active".to_string()), - superseded_by: None, - deprecated_at: None, - review_after: Some((now + chrono::Duration::days(review_days)).format("%Y-%m-%d").to_string()), - }; - - let file_path = memories_add(gcx.clone(), &frontmatter, &content).await?; - let new_doc_id = frontmatter.id.clone().unwrap(); - - let deprecation_candidates = kg.get_deprecation_candidates( - &final_tags, - &final_filenames, - &entities, - Some(&new_doc_id), - ); - - let mut deprecated_count = 0; - if !deprecation_candidates.is_empty() { - let snippet: String = content.chars().take(500).collect(); - - match check_deprecation( - ccx.clone(), - &light_model, - final_title.as_deref().unwrap_or("Untitled"), - &final_tags, - &final_filenames, - &snippet, - &deprecation_candidates, - ).await { - Ok(result) => { - for decision in result.deprecate { - if decision.confidence >= 0.75 { - if let Some(doc) = kg.get_doc_by_id(&decision.target_id) { - if let Err(e) = deprecate_document( - gcx.clone(), - &doc.path, - Some(&new_doc_id), - &decision.reason, - ).await { - warn!("Failed to deprecate {}: {}", decision.target_id, e); - } else { - deprecated_count += 1; - info!("Deprecated {} (confidence: {:.2}): {}", decision.target_id, decision.confidence, decision.reason); - } - } - } - } - } - Err(e) => { - warn!("Deprecation check failed: {}", e); - } - } - } - - let mut result_msg = format!("Knowledge entry created: {}", file_path.display()); - if deprecated_count > 0 { - result_msg.push_str(&format!("\n{} outdated document(s) marked as deprecated.", deprecated_count)); - } + let result_msg = format!("Knowledge entry created: {}", file_path.display()); Ok((false, vec![ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), diff --git a/refact-agent/engine/src/tools/tool_deep_research.rs b/refact-agent/engine/src/tools/tool_deep_research.rs index d75b2c38d..0a12842d0 100644 --- a/refact-agent/engine/src/tools/tool_deep_research.rs +++ b/refact-agent/engine/src/tools/tool_deep_research.rs @@ -9,6 +9,7 @@ use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, Too use crate::call_validation::{ChatMessage, ChatContent, ChatUsage, ContextEnum, SubchatParameters}; use crate::at_commands::at_commands::AtCommandsContext; use crate::integrations::integr_abstract::IntegrationConfirmation; +use crate::memories::{memories_add_enriched, EnrichmentParams}; pub struct ToolDeepResearch { pub config_path: String, @@ -191,6 +192,23 @@ impl Tool for ToolDeepResearch { let final_message = format!("# Deep Research Report\n\n{}", research_result.content.content_text_only()); tracing::info!("Deep research completed"); + let title = if research_query.len() > 80 { + format!("{}...", &research_query[..80]) + } else { + research_query.clone() + }; + let enrichment_params = EnrichmentParams { + base_tags: vec!["research".to_string(), "deep-research".to_string()], + base_filenames: vec![], + base_kind: "research".to_string(), + base_title: Some(title), + }; + if let Err(e) = memories_add_enriched(ccx.clone(), &final_message, enrichment_params).await { + tracing::warn!("Failed to create enriched memory from deep research: {}", e); + } else { + tracing::info!("Created enriched memory from deep research"); + } + let mut results = vec![]; results.push(ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), diff --git a/refact-agent/engine/src/tools/tool_strategic_planning.rs b/refact-agent/engine/src/tools/tool_strategic_planning.rs index ec887ae75..c1aa39b19 100644 --- a/refact-agent/engine/src/tools/tool_strategic_planning.rs +++ b/refact-agent/engine/src/tools/tool_strategic_planning.rs @@ -19,6 +19,7 @@ use crate::files_in_workspace::get_file_text_from_memory_or_disk; use crate::global_context::try_load_caps_quickly_if_not_present; use crate::postprocessing::pp_context_files::postprocess_context_files; use crate::tokens::count_text_tokens_with_fallback; +use crate::memories::{memories_add_enriched, EnrichmentParams}; pub struct ToolStrategicPlanning { pub config_path: String, @@ -318,6 +319,22 @@ impl Tool for ToolStrategicPlanning { let (_, initial_solution) = result?; let final_message = format!("# Solution\n{}", initial_solution.content.content_text_only()); tracing::info!("strategic planning response (combined):\n{}", final_message); + + let filenames: Vec = important_paths.iter() + .map(|p| p.to_string_lossy().to_string()) + .collect(); + let enrichment_params = EnrichmentParams { + base_tags: vec!["planning".to_string(), "strategic".to_string()], + base_filenames: filenames, + base_kind: "decision".to_string(), + base_title: Some("Strategic Plan".to_string()), + }; + if let Err(e) = memories_add_enriched(ccx.clone(), &final_message, enrichment_params).await { + tracing::warn!("Failed to create enriched memory from strategic planning: {}", e); + } else { + tracing::info!("Created enriched memory from strategic planning"); + } + let mut results = vec![]; results.push(ContextEnum::ChatMessage(ChatMessage { role: "tool".to_string(), From 4d8614ad13012beb27333692bd8e75ba814c0428 Mon Sep 17 00:00:00 2001 From: JegernOUTT Date: Wed, 17 Dec 2025 17:05:03 +1030 Subject: [PATCH 8/8] Trigger knowledge graph rebuild in save_trajectory --- refact-agent/engine/src/memories.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/refact-agent/engine/src/memories.rs b/refact-agent/engine/src/memories.rs index c13b9bc4c..499933a14 100644 --- a/refact-agent/engine/src/memories.rs +++ b/refact-agent/engine/src/memories.rs @@ -280,6 +280,8 @@ pub async fn save_trajectory( vecdb.vectorizer_enqueue_files(&vec![file_path.to_string_lossy().to_string()], true).await; } + let _ = build_knowledge_graph(gcx).await; + Ok(file_path) }