fix(ingestion): prevent stdlib contamination by re-resolving version in Tier 1 and validating rustdoc JSON\n\n- Re-resolve (version, url) just before Tier 1 download\n- Proper stdlib handling via construct_stdlib_url; clean fallback on 404\n- Only set tier=rustdoc_json after content sanity check\n- Add ingest re-exports (get_stdlib_url, normalize_item_type, resolve_parent_id)\n- normalize_item_type: preserve unknown string kinds as-is

Author: Peter

.code/agents/352185e2-d05d-4cae-bd90-524c2c34cb87/result.txt

@@ -20,6 +20,8 @@
     batch_examples,
     # Rustdoc parsing
     build_module_hierarchy,
+    normalize_item_type,
+    resolve_parent_id,
     # Cache management
     calculate_cache_size,
     calculate_example_hash,
@@ -29,6 +31,8 @@
     cleanup_embedding_model,
     # Version resolution
     construct_stdlib_url,
+    # Backward-compat alias expected by tests
+    construct_stdlib_url as get_stdlib_url,
     decompress_content,
     download_rustdoc,
     evict_cache_if_needed,
@@ -88,6 +92,8 @@
     "download_rustdoc",
     "decompress_content",
     "construct_stdlib_url",
+    # Backward-compat alias
+    "get_stdlib_url",
     # Rustdoc parsing
     "parse_rustdoc_items_streaming",
     "parse_rustdoc_items",

.code/agents/d57ce572-3055-42b1-8971-299a9c8158a7/result.txt

@@ -37,6 +37,7 @@
     build_module_hierarchy,
     parse_rustdoc_items,
     parse_rustdoc_items_streaming,
+    normalize_item_type,
     resolve_parent_id,
 )
 from .signature_extractor import (
@@ -97,6 +98,7 @@
     "parse_rustdoc_items_streaming",
     "parse_rustdoc_items",
     "build_module_hierarchy",
+    "normalize_item_type",
     "resolve_parent_id",
     # Signature extraction
     "format_signature",

src/docsrs_mcp/ingest.py

@@ -56,6 +56,7 @@
     download_rustdoc,
     fetch_crate_info,
     is_stdlib_crate,
+    construct_stdlib_url,
     resolve_stdlib_version,
     resolve_version,
 )
@@ -556,18 +557,64 @@ async def ingest_crate(crate_name: str, version: str | None = None) -> Path:
                 try:
                     await set_ingestion_status(db_path, crate_id, "downloading")
 
+                    # Always ensure we have a concrete version and URL right before download
+                    tier1_url: str | None = None
+
                     if is_stdlib:
-                        # For stdlib, try docs.rs URL (expected to fail but worth trying)
-                        rustdoc_url = (
-                            f"https://docs.rs/{crate_name}/latest/{crate_name}.json"
+                        # docs.rs generally doesn't host stdlib JSON; construct a throwaway URL
+                        # so that RustdocVersionNotFoundError can drive clean fallback without
+                        # contaminating state.
+                        try:
+                            tier1_url = construct_stdlib_url(crate_name, resolved_version)
+                        except Exception:
+                            tier1_url = None
+                    else:
+                        # Use any previously resolved URL if available; otherwise resolve now
+                        try:
+                            prior_url = rustdoc_url  # may be set earlier during initial resolve
+                        except NameError:
+                            prior_url = None
+
+                        needs_resolution = (
+                            not resolved_version
+                            or str(resolved_version).strip().lower() in ("", "latest")
+                            or not prior_url
                         )
 
+                        if needs_resolution:
+                            try:
+                                resolved_version, tier1_url = await resolve_version(
+                                    session, crate_name, version
+                                )
+                            except Exception as e:
+                                logger.info(
+                                    f"Tier1: resolve_version retry skipped/failed for {crate_name}@{version}: {e}"
+                                )
+                                tier1_url = prior_url
+                        else:
+                            tier1_url = prior_url
+
+                    # Final guards before attempting download
+                    if not isinstance(resolved_version, str) or not resolved_version:
+                        resolved_version = version or "latest"
+
+                    logger.info(
+                        f"Tier1: attempting rustdoc download crate={crate_name} resolved_version={resolved_version} url={tier1_url} stdlib={is_stdlib}"
+                    )
+
                     raw_content, used_url = await download_rustdoc(
-                        session, crate_name, resolved_version, rustdoc_url
+                        session, crate_name, resolved_version, tier1_url
                     )
 
                     json_content = await decompress_content(raw_content, used_url)
 
+                    # Sanity check before marking tier success to avoid contamination
+                    if not json_content or (
+                        isinstance(json_content, str)
+                        and ('"index"' not in json_content and '"paths"' not in json_content)
+                    ):
+                        raise Exception("Rustdoc JSON content appears invalid/empty")
+
                     ingestion_tier = IngestionTier.RUSTDOC_JSON
 
                     # Parse and store

src/docsrs_mcp/ingestion/__init__.py

@@ -524,7 +524,8 @@ def normalize_item_type(kind: dict | str) -> str:
         if key in kind_str:
             return value
 
-    return "unknown"
+    # Preserve unknown string kinds as-is; use "unknown" for dicts or empty
+    return kind_str if not isinstance(kind, dict) else "unknown"
 
 
 def extract_item_path(item: dict) -> str: