Update: Port for Sphinx 8.x.x, fix path naming and Cargo.toml compatibility

.readthedocs.yaml

@@ -4,6 +4,7 @@
 version: 2
 
 sphinx:
+  configuration: docs/conf.py
   builder: html
 
 build:

crates/analyzer/Cargo.toml

@@ -20,6 +20,7 @@ serde.workspace = true
 serde_json.workspace = true
 syn.workspace = true
 toml.workspace = true
+cargo_metadata = "0.18"
 
 [dev-dependencies]
 insta.workspace = true

crates/analyzer/src/analyze/crate_.rs

@@ -1,79 +1,76 @@
 //! Analyze the crate
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
+use cargo_metadata::{MetadataCommand, Target};
 use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
 
 use crate::data_model::{Crate, Enum, Function, Module, Struct};
 
 pub fn analyze_crate(path: &str) -> Result<AnalysisResult> {
     // make the path absolute
     // TODO we use dunce to canonicalize the path because otherwise there is issues with python's os.path.relpath on windows, but maybe we should fix this on the Python side
-    let path =
+    let crate_dir =
         dunce::canonicalize(path).context(format!("Error resolving crate path: {}", path))?;
+    eprintln!("running new analyzer");
     // check the path is a directory
-    if !path.is_dir() {
-        return Err(anyhow::anyhow!(format!(
+    if !crate_dir.is_dir() {
+        return Err(anyhow!(
             "Crate path is not a directory: {}",
-            path.to_string_lossy()
-        )));
+            crate_dir.to_string_lossy()
+        ));
     }
     // check if Cargo.toml exists
-    let cargo_toml_path = path.join("Cargo.toml");
+    let cargo_toml_path = crate_dir.join("Cargo.toml");
     if !cargo_toml_path.exists() {
-        return Err(anyhow::anyhow!(format!(
+        return Err(anyhow!(
             "Cargo.toml does not exist in: {}",
-            path.to_string_lossy()
-        )));
+            crate_dir.to_string_lossy()
+        ));
     }
 
-    // read the Cargo.toml and initialize the Crate struct
-    let contents = std::fs::read_to_string(&cargo_toml_path)?;
-    let cargo_toml: CargoToml = toml::from_str(&contents).context(format!(
-        "Error parsing: {}",
-        cargo_toml_path.to_string_lossy()
-    ))?;
+    // use `cargo_metadata` instead of implementing own TOML parser
+    let metadata = MetadataCommand::new()
+        .manifest_path(&cargo_toml_path)
+        .exec()
+        .context("Failed to run `cargo metadata`")?;
 
-    // check whether the crate is a library or binary
-    let (crate_name, to_root) = if let Some(lib) = cargo_toml.lib {
-        if cargo_toml.bin.is_some() {
-            return Err(anyhow::anyhow!(format!(
-                "Both lib and bin sections in: {}",
-                path.to_string_lossy()
-            )));
-        }
-        (
-            lib.name.unwrap_or(cargo_toml.package.name),
-            lib.path.unwrap_or("src/lib.rs".to_string()),
-        )
-    } else if let Some(bin) = cargo_toml.bin {
-        (
-            bin.name.unwrap_or(cargo_toml.package.name),
-            bin.path.unwrap_or("src/main.rs".to_string()),
-        )
-    } else {
-        return Err(anyhow::anyhow!(format!(
-            "No lib or bin section in: {}",
-            path.to_string_lossy()
-        )));
-    };
+    let root_pkg = metadata
+        .root_package()
+        .ok_or_else(|| anyhow!("`cargo metadata` returned no root package"))?;
+
+    // Prefer library target; fall back to the first binary target
+    let root_target: &Target = root_pkg
+        .targets
+        .iter()
+        .find(|t| t.kind.contains(&"lib".into()))
+        .or_else(|| root_pkg.targets.iter().find(|t| t.kind.contains(&"bin".into())))
+        .ok_or_else(|| anyhow!("No lib or bin target defined in manifest"))?;
+
+    let crate_name = root_target.name.clone();
+    let root_module = PathBuf::from(&root_target.src_path);
 
     let mut result = AnalysisResult::new(Crate {
-        name: crate_name,
-        version: cargo_toml.package.version.clone(),
+        name: crate_name.clone(),
+        version: root_pkg.version.to_string(), // workspace-aware
     });
 
     // check existence of the root module
-    let root_module = path.join(to_root);
     if !root_module.exists() {
         return Ok(result);
     }
 
     // read the top-level module
     let content = std::fs::read_to_string(&root_module)?;
-    let (module, structs, enums, functions) =
-        Module::parse(Some(&root_module), &[&result.crate_.name], &content).context(format!(
-            "Error parsing module {}",
-            root_module.to_string_lossy()
-        ))?;
+    let (module, structs, enums, functions) = Module::parse(
+        Some(&root_module),
+        &[&result.crate_.name],
+        &content,
+    )
+    .context(format!(
+        "Error parsing module {}",
+        root_module.to_string_lossy()
+    ))?;
+
     let mut modules_to_read = module
         .declarations
         .iter()
@@ -91,24 +88,23 @@ pub fn analyze_crate(path: &str) -> Result<AnalysisResult> {
     result.enums.extend(enums);
     result.functions.extend(functions);
 
-    // recursively find/read the public sub-modules
+    // recursively find/read the public sub‑modules
     let mut read_modules = vec![];
     while let Some((parent_dir, module_name, parent)) = modules_to_read.pop() {
-        let (module_path, submodule_dir) =
-            if parent_dir.join(&module_name).with_extension("rs").exists() {
-                (
-                    parent_dir.join(&module_name).with_extension("rs"),
-                    parent_dir.join(&module_name),
-                )
-            } else if parent_dir.join(&module_name).join("mod.rs").exists() {
-                (
-                    parent_dir.join(&module_name).join("mod.rs"),
-                    parent_dir.to_path_buf(),
-                )
-            } else {
-                // TODO warn about missing module?
-                continue;
-            };
+        let (module_path, submodule_dir) = if parent_dir.join(&module_name).with_extension("rs").exists() {
+            (
+                parent_dir.join(&module_name).with_extension("rs"),
+                parent_dir.join(&module_name),
+            )
+        } else if parent_dir.join(&module_name).join("mod.rs").exists() {
+            (
+                parent_dir.join(&module_name).join("mod.rs"),
+                parent_dir.to_path_buf(),
+            )
+        } else {
+            // TODO warn about missing module?
+            continue;
+        };
 
         if read_modules.contains(&module_path) {
             continue;
@@ -126,12 +122,12 @@ pub fn analyze_crate(path: &str) -> Result<AnalysisResult> {
             "Error parsing module {}",
             module_path.to_string_lossy()
         ))?;
+
         modules_to_read.extend(
             module
                 .declarations
                 .iter()
-                .map(|s| (submodule_dir.clone(), s.to_string(), path.clone()))
-                .collect::<Vec<_>>(),
+                .map(|s| (submodule_dir.clone(), s.to_string(), path.clone())),
         );
         result.modules.push(module);
         result.structs.extend(structs);
@@ -164,31 +160,6 @@ impl AnalysisResult {
     }
 }
 
-#[derive(Debug, Deserialize)]
-struct CargoToml {
-    package: Package,
-    bin: Option<Bin>,
-    lib: Option<Lib>,
-}
-
-#[derive(Debug, Deserialize)]
-struct Package {
-    name: String,
-    version: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct Lib {
-    name: Option<String>,
-    path: Option<String>,
-}
-
-#[derive(Debug, Deserialize)]
-struct Bin {
-    name: Option<String>,
-    path: Option<String>,
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/py_binding/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sphinx_rust"
-version = "0.0.2"
+version = "0.0.2-dev1"
 publish = false
 edition = "2021"
 authors.workspace = true

pyproject.toml

@@ -37,7 +37,7 @@ classifiers = [
   "Topic :: Text Processing :: Markup",
 ]
 dependencies = [
-  "sphinx~=7.3"
+  "sphinx>=7.3"
 ]
 
 [project.urls]

python/sphinx_rust/directives/_core.py

@@ -146,8 +146,11 @@ def parse_docstring(
         return []
 
     docstring = item.docstring if docstring is None else docstring
-    source_path = env.doc2path(  # TODO this actually should be the rust file path
-        env.docname
+
+    source_path = str(
+        env.doc2path(  # TODO this actually should be the rust file path
+            env.docname
+        )
     )
     # TODO how to handle line numbers?
     document = utils.new_document(source_path, doc.settings)

python/sphinx_rust/domain.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 import shutil
 from typing import TYPE_CHECKING, Literal, TypedDict
+import re
 
 from sphinx import addnodes
 from sphinx.domains import Domain
@@ -20,6 +21,19 @@
 from sphinx_rust.directives.struct import RustStructAutoDirective
 from sphinx_rust.sphinx_rust import analyze_crate, load_descendant_modules
 
+
+INVALID_CHARS = r"[^A-Za-z0-9._-]"
+
+
+def slugify_rust_name(fullname: str) -> Path:
+    """
+    Turn `crate::mod::Type<T>` into a safe relative Path:
+    crate/mod/Type_T
+    """
+    parts = fullname.split("::")
+    cleaned = [re.sub(INVALID_CHARS, "_", p) for p in parts]
+    return Path(*cleaned)
+
 if TYPE_CHECKING:
     from docutils.nodes import Element
     from sphinx.addnodes import pending_xref
@@ -225,17 +239,31 @@ def create_pages(srcdir: Path, result: AnalysisResult) -> None:
 
 
 def create_object_pages(folder: Path, otype: str, names: list[str]) -> None:
-    """Create the pages for the objects of a certain type."""
-    ofolder = folder.joinpath(otype + "s")
+    ofolder = folder / f"{otype}s"
     ofolder.mkdir(exist_ok=True)
-    index_content = f"{otype.capitalize()}s\n{'=' * (len(otype) + 1)}\n\n.. toctree::\n    :maxdepth: 1\n\n"
-    for name in names:
-        index_content += f"    {name}\n"
-        title = f"{otype.capitalize()} ``{name}``"
-        ofolder.joinpath(f"{name}.rst").write_text(
-            f"{title}\n{'=' * len(title)}\n\n.. rust:{otype}:: {name}\n"
+
+    idx_lines = [
+        f"{otype.capitalize()}s",
+        "=" * (len(otype) + 1),
+        "",
+        ".. toctree::",
+        "    :maxdepth: 1",
+        ""
+    ]
+
+    for rust_name in names:
+        rel_path = slugify_rust_name(rust_name)
+        idx_lines.append(f"    {rel_path.as_posix()}")
+
+        dst_file = ofolder / rel_path.with_suffix(".rst")
+        dst_file.parent.mkdir(parents=True, exist_ok=True)
+
+        title = f"{otype.capitalize()} ``{rust_name}``"
+        dst_file.write_text(
+            f"{title}\n{'=' * len(title)}\n\n.. rust:{otype}:: {rust_name}\n"
         )
-    ofolder.joinpath("index.rst").write_text(index_content)
+
+    (ofolder / "index.rst").write_text("\n".join(idx_lines) + "\n")
 
 
 def create_code_pages(crate_name: str, srcdir: Path, cache: Path) -> None:
@@ -247,20 +275,17 @@ def create_code_pages(crate_name: str, srcdir: Path, cache: Path) -> None:
         code_folder = srcdir.joinpath("api", "crates", crate_name, "code")
         code_folder.mkdir(exist_ok=True, parents=True)
         for full_name, file_path in modules:
-            # TODO catch exceptions here, if a relative path cannot be created
-            rel_path = os.path.relpath(Path(file_path), code_folder)
-            # note, this is available only in Python 3.12+
-            # rel_path = Path(file_path).relative_to(code_folder, walk_up=True)
-            # TODO only write the file if it doesn't exist or is different
-            code_folder.joinpath(f"{full_name}.rst").write_text(
-                "\n".join(
-                    (
-                        ":orphan:",
-                        "",
-                        f".. literalinclude:: {rel_path}",
-                        f"   :name: rust-code:{full_name}",
-                        "   :language: rust",
-                        "   :linenos:",
-                    )
-                )
+            rel = slugify_rust_name(full_name)
+            dst = code_folder / rel.with_suffix(".rst")
+            dst.parent.mkdir(parents=True, exist_ok=True)
+
+            dst.write_text(
+                "\n".join((
+                    ":orphan:",
+                    "",
+                    f".. literalinclude:: {os.path.relpath(file_path, dst.parent)}",
+                    f"   :name: rust-code:{full_name}",
+                    "   :language: rust",
+                    "   :linenos:",
+                ))
             )