Improve tools

  1. Fixed the lookup_item command to properly handle item paths by:
    - Trying different item types (struct, enum, trait, fn, macro)
    - Using the proper URL structure for docs.rs items
    - Adding User-Agent headers to avoid 404 errors
  2. Fixed the search_crates command to:
    - Include proper User-Agent headers to avoid 403 Forbidden errors
    - Return JSON data in a consistent format
  3. Made general improvements:
    - Better CLI help text with examples
    - Better error messages with specific examples
    - More comprehensive documentation of usage patterns

Author: Danielle Jenkins

src/bin/cratedocs.rs

@@ -159,10 +159,14 @@ async fn run_test_tool(
         println!("Usage examples:");
         println!("  cargo run --bin cratedocs -- test --tool lookup_crate --crate-name serde");
         println!("  cargo run --bin cratedocs -- test --tool lookup_crate --crate-name tokio --version 1.35.0");
+        println!("  cargo run --bin cratedocs -- test --tool lookup_item --crate-name tokio --item-path sync::mpsc::Sender");
+        println!("  cargo run --bin cratedocs -- test --tool lookup_item --crate-name serde --item-path Serialize --version 1.0.147");
         println!("  cargo run --bin cratedocs -- test --tool search_crates --query logger\n");
         println!("Available tools:");
         println!("  lookup_crate   - Look up documentation for a Rust crate");
         println!("  lookup_item    - Look up documentation for a specific item in a crate");
+        println!("                   Format: 'module::path::ItemName' (e.g., 'sync::mpsc::Sender')");
+        println!("                   The tool will try to detect if it's a struct, enum, trait, fn, or macro");
         println!("  search_crates  - Search for crates on crates.io");
         println!("  help           - Show this help information\n");
         return Ok(());
@@ -224,9 +228,12 @@ async fn run_test_tool(
         Ok(result) => result,
         Err(e) => {
             eprintln!("\nERROR: {}", e);
-            eprintln!("\nTip: The direct item lookup may require very specific path formats. Try these commands instead:");
+            eprintln!("\nTip: Try these suggestions:");
             eprintln!("  - For crate docs: cargo run --bin cratedocs -- test --tool lookup_crate --crate-name tokio");
-            eprintln!("  - For crate docs with version: cargo run --bin cratedocs -- test --tool lookup_crate --crate-name serde --version 1.0.147");
+            eprintln!("  - For item lookup: cargo run --bin cratedocs -- test --tool lookup_item --crate-name tokio --item-path sync::mpsc::Sender");
+            eprintln!("  - For item lookup with version: cargo run --bin cratedocs -- test --tool lookup_item --crate-name serde --item-path Serialize --version 1.0.147");
+            eprintln!("  - For crate search: cargo run --bin cratedocs -- test --tool search_crates --query logger --limit 5");
+            eprintln!("  - For help: cargo run --bin cratedocs -- test --tool help");
             return Ok(());
         }
     };

src/tools/docs/docs.rs

@@ -83,9 +83,13 @@ impl DocRouter {
         };
 
         // Fetch the documentation page
-        let response = self.client.get(&url).send().await.map_err(|e| {
-            ToolError::ExecutionError(format!("Failed to fetch documentation: {}", e))
-        })?;
+        let response = self.client.get(&url)
+            .header("User-Agent", "CrateDocs/0.1.0 (https://github.com/d6e/cratedocs-mcp)")
+            .send()
+            .await
+            .map_err(|e| {
+                ToolError::ExecutionError(format!("Failed to fetch documentation: {}", e))
+            })?;
 
         if !response.status().is_success() {
             return Err(ToolError::ExecutionError(format!(
@@ -113,9 +117,13 @@ impl DocRouter {
 
         let url = format!("https://crates.io/api/v1/crates?q={}&per_page={}", query, limit);
 
-        let response = self.client.get(&url).send().await.map_err(|e| {
-            ToolError::ExecutionError(format!("Failed to search crates.io: {}", e))
-        })?;
+        let response = self.client.get(&url)
+            .header("User-Agent", "CrateDocs/0.1.0 (https://github.com/d6e/cratedocs-mcp)")
+            .send()
+            .await
+            .map_err(|e| {
+                ToolError::ExecutionError(format!("Failed to search crates.io: {}", e))
+            })?;
 
         if !response.status().is_success() {
             return Err(ToolError::ExecutionError(format!(
@@ -151,36 +159,78 @@ impl DocRouter {
             return Ok(doc);
         }
 
-        // Construct the docs.rs URL for the specific item
-        let url = if let Some(ver) = version {
-            format!("https://docs.rs/{}/{}/{}/", crate_name, ver, item_path.replace("::", "/"))
+        // Process the item path to determine the item type
+        // Format: module::path::ItemName
+        // Need to split into module path and item name, and guess item type
+        let parts: Vec<&str> = item_path.split("::").collect();
+        
+        if parts.is_empty() {
+            return Err(ToolError::InvalidParameters(
+                "Invalid item path. Expected format: module::path::ItemName".to_string()
+            ));
+        }
+        
+        let item_name = parts.last().unwrap().to_string();
+        let module_path = if parts.len() > 1 {
+            parts[..parts.len()-1].join("/")
         } else {
-            format!("https://docs.rs/{}/latest/{}/", crate_name, item_path.replace("::", "/"))
+            String::new()
         };
-
-        // Fetch the documentation page
-        let response = self.client.get(&url).send().await.map_err(|e| {
-            ToolError::ExecutionError(format!("Failed to fetch item documentation: {}", e))
-        })?;
-
-        if !response.status().is_success() {
-            return Err(ToolError::ExecutionError(format!(
-                "Failed to fetch item documentation. Status: {}",
-                response.status()
-            )));
-        }
-
-        let html_body = response.text().await.map_err(|e| {
-            ToolError::ExecutionError(format!("Failed to read response body: {}", e))
-        })?;
 
-        // Convert HTML to markdown
-        let markdown_body = parse_html(&html_body);
-
-        // Cache the markdown result
-        self.cache.set(cache_key, markdown_body.clone()).await;
+        // Try different item types (struct, enum, trait, fn)
+        let item_types = ["struct", "enum", "trait", "fn", "macro"];
+        let mut last_error = None;
 
-        Ok(markdown_body)
+        for item_type in item_types.iter() {
+            // Construct the docs.rs URL for the specific item
+            let url = if let Some(ver) = version.clone() {
+                if module_path.is_empty() {
+                    format!("https://docs.rs/{}/{}/{}/{}.{}.html", crate_name, ver, crate_name, item_type, item_name)
+                } else {
+                    format!("https://docs.rs/{}/{}/{}/{}/{}.{}.html", crate_name, ver, crate_name, module_path, item_type, item_name)
+                }
+            } else {
+                if module_path.is_empty() {
+                    format!("https://docs.rs/{}/latest/{}/{}.{}.html", crate_name, crate_name, item_type, item_name)
+                } else {
+                    format!("https://docs.rs/{}/latest/{}/{}/{}.{}.html", crate_name, crate_name, module_path, item_type, item_name)
+                }
+            };
+            
+            // Try to fetch the documentation page
+            let response = match self.client.get(&url)
+                .header("User-Agent", "CrateDocs/0.1.0 (https://github.com/d6e/cratedocs-mcp)")
+                .send().await {
+                Ok(resp) => resp,
+                Err(e) => {
+                    last_error = Some(e.to_string());
+                    continue;
+                }
+            };
+            
+            // If found, process and return
+            if response.status().is_success() {
+                let html_body = response.text().await.map_err(|e| {
+                    ToolError::ExecutionError(format!("Failed to read response body: {}", e))
+                })?;
+                
+                // Convert HTML to markdown
+                let markdown_body = parse_html(&html_body);
+                
+                // Cache the markdown result
+                self.cache.set(cache_key, markdown_body.clone()).await;
+                
+                return Ok(markdown_body);
+            }
+            
+            last_error = Some(format!("Status code: {}", response.status()));
+        }
+        
+        // If we got here, none of the item types worked
+        Err(ToolError::ExecutionError(format!(
+            "Failed to fetch item documentation. No matching item found. Last error: {}",
+            last_error.unwrap_or_else(|| "Unknown error".to_string())
+        )))
     }
 }