tdlr didn't work with stdio

Author: elasticdotventures

README.md

@@ -41,15 +41,16 @@ cargo run --bin cratedocs http --debug
 
 ### Directly Testing Documentation Tools
 
-# Enumerate crate items
-cargo run --bin cratedocs test --tool list_crate_items --crate-name serde --version 1.0.0 --item-type struct
-cargo run --bin cratedocs test --tool list_crate_items --crate-name tokio --version 1.28.0 --visibility pub --module tokio::sync
 You can directly test the documentation tools from the command line without starting a server:
 
 ```bash
 # Get help for the test command
 cargo run --bin cratedocs test --tool help
 
+# Enumerate crate items (step by step)
+cargo run --bin cratedocs test --tool list_crate_items --crate-name serde --version 1.0.0 --item-type struct
+cargo run --bin cratedocs test --tool list_crate_items --crate-name tokio --version 1.28.0 --visibility pub --module tokio::sync
+
 # Look up crate documentation
 cargo run --bin cratedocs test --tool lookup_crate --crate-name tokio
 
@@ -74,6 +75,9 @@ cargo run --bin cratedocs test --tool lookup_crate --crate-name tokio --output t
 
 # Summarize output by stripping LICENSE and VERSION sections, limits to xxxxx tokens (uses huggingface tokenizer)
 cargo run --bin cratedocs test --tool lookup_crate --crate-name tokio --tldr --max_tokens 48000
+
+
+
 ```
 
 By default, the HTTP server will listen on `http://127.0.0.1:8080/sse`.

src/bin/cratedocs.rs

@@ -9,7 +9,7 @@ use std::net::SocketAddr;
 use tokio::io::{stdin, stdout};
 use tracing_appender::rolling::{RollingFileAppender, Rotation};
 use tracing_subscriber::{self, EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
-use regex::Regex;
+use cratedocs_mcp::tools::tldr;
 
 #[derive(Parser)]
 #[command(author, version = "0.2.0", about, long_about = None)]
@@ -29,6 +29,9 @@ enum Commands {
         /// Enable debug logging
         #[arg(short, long)]
         debug: bool,
+        /// Summarize output by stripping LICENSE and VERSION sections (TL;DR mode)
+        #[arg(long)]
+        tldr: bool,
     },
     /// Run the server with HTTP/SSE interface
     Http {
@@ -109,7 +112,7 @@ async fn main() -> Result<()> {
             println!("{}", env!("CARGO_PKG_VERSION"));
             Ok(())
         },
-        Commands::Stdio { debug } => run_stdio_server(debug).await,
+        Commands::Stdio { debug, tldr } => run_stdio_server(debug, tldr).await,
         Commands::Http { address, debug } => run_http_server(address, debug).await,
         Commands::Test {
             tool,
@@ -145,7 +148,7 @@ async fn main() -> Result<()> {
     }
 }
 
-async fn run_stdio_server(debug: bool) -> Result<()> {
+async fn run_stdio_server(debug: bool, tldr: bool) -> Result<()> {
     // Set up file appender for logging
     let file_appender = RollingFileAppender::new(Rotation::DAILY, "logs", "stdio-server.log");
 
@@ -164,13 +167,16 @@ async fn run_stdio_server(debug: bool) -> Result<()> {
     tracing::info!("Starting MCP documentation server in STDIN/STDOUT mode");
 
     // Create an instance of our documentation router
-    let router = RouterService(DocRouter::new());
+    // If tldr is needed globally, you may want to pass it to DocRouter or handle it in tool output
+    let router = RouterService(DocRouter::new_with_tldr(tldr));
 
     // Create and run the server
     let server = Server::new(router);
     let transport = ByteTransport::new(stdin(), stdout());
 
     tracing::info!("Documentation server initialized and ready to handle requests");
+    // Note: tldr is parsed and available, but not yet used in stdio mode.
+    // If you want to apply TLDR globally, you would need to modify DocRouter or Server to use it.
     Ok(server.run(transport).await?)
 }
 
@@ -201,36 +207,6 @@ async fn run_http_server(address: String, debug: bool) -> Result<()> {
 }
 
 // --- TLDR Helper Function ---
-fn apply_tldr(input: &str) -> String {
-    // Remove LICENSE and VERSION(S) sections by skipping lines between those headings and the next heading or EOF.
-    let mut output = Vec::new();
-    let mut skip = false;
-
-    // Match any heading (with or without space) for LICENSE or VERSION(S)
-    let tldr_section_re = Regex::new(r"(?i)^\s*#+\s*license\b|^\s*#+\s*version(s)?\b|^\s*#+license\b|^\s*#+version(s)?\b").unwrap();
-    // Match any heading (for ending the skip)
-    let heading_re = Regex::new(r"^\s*#+").unwrap();
-    // Match <detail> tags including start, end, and inline attributes
-    let detail_tag_re = Regex::new(r"<[/]?detail.*?>").unwrap();
-
-    for line in input.lines() {
-        // Start skipping if we hit a LICENSE or VERSION(S) heading
-        if !skip && tldr_section_re.is_match(line) {
-            skip = true;
-            continue; // skip the heading line itself
-        }
-        // Stop skipping at the next heading (but do not skip the heading itself)
-        if skip && heading_re.is_match(line) && !tldr_section_re.is_match(line) {
-            skip = false;
-        }
-        if !skip {
-            // Remove <detail> tags from the line
-            let cleaned_line = detail_tag_re.replace_all(line, "").to_string();
-            output.push(cleaned_line.to_string());
-        }
-    }
-    output.iter().map(|s| s.as_str()).collect::<Vec<_>>().join("\n")
-}
 
 /// Configuration for the test tool
 struct TestToolConfig {
@@ -407,7 +383,7 @@ async fn run_test_tool(config: TestToolConfig) -> Result<()> {
 
                 // TL;DR processing: strip LICENSE and VERSION(S) sections if --tldr is set
                 if tldr {
-                    content_str = apply_tldr(&content_str);
+                    content_str = tldr::apply_tldr(&content_str);
                 }
 
                 let formatted_output = match format.as_str() {
@@ -507,7 +483,7 @@ async fn run_test_tool(config: TestToolConfig) -> Result<()> {
 }
 #[cfg(test)]
 mod tldr_tests {
-    use super::apply_tldr;
+    use cratedocs_mcp::tools::tldr::apply_tldr;
 
     #[test]
     fn test_apply_tldr_removes_license_and_versions() {

src/tools/docs/docs.rs

@@ -1,4 +1,5 @@
 use crate::tools::item_list;
+use crate::tools::tldr;
 use std::{future::Future, pin::Pin, sync::Arc};
 
 use mcp_core::{
@@ -47,6 +48,7 @@ impl DocCache {
 pub struct DocRouter {
     pub client: Client,
     pub cache: DocCache,
+    pub tldr: bool,
 }
 
 impl Default for DocRouter {
@@ -56,12 +58,16 @@ impl Default for DocRouter {
 }
 
 impl DocRouter {
-    pub fn new() -> Self {
+    pub fn new_with_tldr(tldr: bool) -> Self {
         Self {
             client: Client::new(),
             cache: DocCache::new(),
+            tldr,
         }
     }
+    pub fn new() -> Self {
+        Self::new_with_tldr(false)
+    }
 
     // Fetch crate documentation from docs.rs
     async fn lookup_crate(&self, crate_name: String, version: Option<String>) -> Result<String, ToolError> {
@@ -363,9 +369,10 @@ impl mcp_server::Router for DocRouter {
         let this = self.clone();
         let tool_name = tool_name.to_string();
         let arguments = arguments.clone();
+        let tldr = self.tldr;
 
         Box::pin(async move {
-            match tool_name.as_str() {
+            let mut result = match tool_name.as_str() {
                 "lookup_crate" => {
                     let crate_name = arguments
                         .get("crate_name")
@@ -455,7 +462,18 @@ impl mcp_server::Router for DocRouter {
                     Ok(vec![Content::text(result)])
                 }
                 _ => Err(ToolError::NotFound(format!("Tool {} not found", tool_name))),
+            }?;
+
+            // Apply TLDR filter if enabled
+            if tldr {
+                for content in &mut result {
+                    if let Content::Text(text) = content {
+                        text.text = tldr::apply_tldr(&text.text);
+                    }
+                }
             }
+
+            Ok(result)
         })
     }
 

src/tools/item_list.rs

@@ -4,7 +4,7 @@ use std::fs;
 use std::path::Path;
 use tar::Archive;
 use flate2::read::GzDecoder;
-use syn::{File, Item};
+use syn::{Item};
 
 /// Represents filters for item listing.
 #[derive(Debug)]

src/tools/mod.rs

@@ -1,5 +1,6 @@
 pub mod docs;
 pub mod item_list;
+pub mod tldr;
 
 pub use docs::DocRouter;
 pub use docs::docs::DocCache;