diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index af81e7aea69..8e7367bd119 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -18,8 +18,10 @@ pub mod kw { syn::custom_keyword!(cancel_handle); syn::custom_keyword!(constructor); syn::custom_keyword!(dict); + syn::custom_keyword!(doc_mode); syn::custom_keyword!(eq); syn::custom_keyword!(eq_int); + syn::custom_keyword!(end_doc_mode); syn::custom_keyword!(extends); syn::custom_keyword!(freelist); syn::custom_keyword!(from_py_with); @@ -318,6 +320,7 @@ pub type StrFormatterAttribute = OptionalKeywordAttribute; pub type SubmoduleAttribute = kw::submodule; pub type GILUsedAttribute = KeywordAttribute; +pub type DocModeAttribute = KeywordAttribute; impl Parse for KeywordAttribute { fn parse(input: ParseStream<'_>) -> Result { diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 10aae4f4810..fd516856cd7 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -975,7 +975,7 @@ impl<'a> FnSpec<'a> { } /// Forwards to [utils::get_doc] with the text signature of this spec. - pub fn get_doc(&self, attrs: &[syn::Attribute], ctx: &Ctx) -> PythonDoc { + pub fn get_doc(&self, attrs: &mut Vec, ctx: &Ctx) -> syn::Result { let text_signature = self .text_signature_call_signature() .map(|sig| format!("{}{}", self.python_name, sig)); diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index e53c35c465b..bc88da8b80e 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -112,7 +112,7 @@ pub fn pymodule_module_impl( options.take_pyo3_options(attrs)?; let ctx = &Ctx::new(&options.krate, None); let Ctx { pyo3_path, .. } = ctx; - let doc = get_doc(attrs, None, ctx); + let doc = get_doc(attrs, None, ctx)?; let name = options .name .map_or_else(|| ident.unraw(), |name| name.value.0); @@ -434,7 +434,7 @@ pub fn pymodule_function_impl( .name .map_or_else(|| ident.unraw(), |name| name.value.0); let vis = &function.vis; - let doc = get_doc(&function.attrs, None, ctx); + let doc = get_doc(&mut function.attrs, None, ctx)?; let initialization = module_initialization( &name, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 98c78e3cf91..0d0a1a4d58c 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -250,7 +250,7 @@ pub fn build_py_class( args.options.take_pyo3_options(&mut class.attrs)?; let ctx = &Ctx::new(&args.options.krate, None); - let doc = utils::get_doc(&class.attrs, None, ctx); + let doc = utils::get_doc(&mut class.attrs, None, ctx)?; if let Some(lt) = class.generics.lifetimes().next() { bail_spanned!( @@ -436,11 +436,11 @@ fn impl_class( ctx, )?; - let (default_class_geitem, default_class_geitem_method) = - pyclass_class_geitem(&args.options, &syn::parse_quote!(#cls), ctx)?; + let (default_class_getitem, default_class_getitem_method) = + pyclass_class_getitem(&args.options, &syn::parse_quote!(#cls), ctx)?; - if let Some(default_class_geitem_method) = default_class_geitem_method { - default_methods.push(default_class_geitem_method); + if let Some(default_class_getitem_method) = default_class_getitem_method { + default_methods.push(default_class_getitem_method); } let (default_str, default_str_slot) = @@ -474,7 +474,7 @@ fn impl_class( #default_richcmp #default_hash #default_str - #default_class_geitem + #default_class_getitem } }) } @@ -521,7 +521,7 @@ pub fn build_py_enum( bail_spanned!(generic.span() => "enums do not support #[pyclass(generic)]"); } - let doc = utils::get_doc(&enum_.attrs, None, ctx); + let doc = utils::get_doc(&mut enum_.attrs, None, ctx)?; let enum_ = PyClassEnum::new(enum_)?; impl_enum(enum_, &args, doc, method_type, ctx) } @@ -1759,7 +1759,7 @@ fn complex_enum_variant_field_getter<'a>( let property_type = crate::pymethod::PropertyType::Function { self_type: &self_type, spec: &spec, - doc: crate::get_doc(&[], None, ctx), + doc: PythonDoc::empty(ctx), }; let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type, ctx)?; @@ -2027,7 +2027,7 @@ fn pyclass_hash( } } -fn pyclass_class_geitem( +fn pyclass_class_getitem( options: &PyClassPyO3Options, cls: &syn::Type, ctx: &Ctx, @@ -2036,7 +2036,7 @@ fn pyclass_class_geitem( match options.generic { Some(_) => { let ident = format_ident!("__class_getitem__"); - let mut class_geitem_impl: syn::ImplItemFn = { + let mut class_getitem_impl: syn::ImplItemFn = { parse_quote! { #[classmethod] fn #ident<'py>( @@ -2049,19 +2049,19 @@ fn pyclass_class_geitem( }; let spec = FnSpec::parse( - &mut class_geitem_impl.sig, - &mut class_geitem_impl.attrs, + &mut class_getitem_impl.sig, + &mut class_getitem_impl.attrs, Default::default(), )?; - let class_geitem_method = crate::pymethod::impl_py_method_def( + let class_getitem_method = crate::pymethod::impl_py_method_def( cls, &spec, - &spec.get_doc(&class_geitem_impl.attrs, ctx), + &spec.get_doc(&mut class_getitem_impl.attrs, ctx)?, Some(quote!(#pyo3_path::ffi::METH_CLASS)), ctx, )?; - Ok((Some(class_geitem_impl), Some(class_geitem_method))) + Ok((Some(class_getitem_impl), Some(class_getitem_method))) } None => Ok((None, None)), } diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 5731cb8f510..91b6d87b2c7 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -424,7 +424,7 @@ pub fn impl_wrap_pyfunction( ); } let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; - let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs, ctx), ctx); + let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&mut func.attrs, ctx)?, ctx); let wrapped_pyfunction = quote! { // Create a module with the same name as the `#[pyfunction]` - this way `use ` diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 6c111774ced..ea1b0b7bd8b 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -136,7 +136,7 @@ pub fn impl_methods( let method = PyMethod::parse(&mut meth.sig, &mut meth.attrs, fun_options)?; #[cfg(feature = "experimental-inspect")] extra_fragments.push(method_introspection_code(&method.spec, ty, ctx)); - match pymethod::gen_py_method(ty, method, &meth.attrs, ctx)? { + match pymethod::gen_py_method(ty, method, &mut meth.attrs, ctx)? { GeneratedPyMethod::Method(MethodAndMethodDef { associated_method, method_def, diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 70aa813b143..20423ffb85a 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -191,7 +191,7 @@ pub fn is_proto_method(name: &str) -> bool { pub fn gen_py_method( cls: &syn::Type, method: PyMethod<'_>, - meth_attrs: &[syn::Attribute], + meth_attrs: &mut Vec, ctx: &Ctx, ) -> Result { let spec = &method.spec; @@ -236,21 +236,21 @@ pub fn gen_py_method( (_, FnType::Fn(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs, ctx), + &spec.get_doc(meth_attrs, ctx)?, None, ctx, )?), (_, FnType::FnClass(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs, ctx), + &spec.get_doc(meth_attrs, ctx)?, Some(quote!(#pyo3_path::ffi::METH_CLASS)), ctx, )?), (_, FnType::FnStatic) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs, ctx), + &spec.get_doc(meth_attrs, ctx)?, Some(quote!(#pyo3_path::ffi::METH_STATIC)), ctx, )?), @@ -264,7 +264,7 @@ pub fn gen_py_method( PropertyType::Function { self_type, spec, - doc: spec.get_doc(meth_attrs, ctx), + doc: spec.get_doc(meth_attrs, ctx)?, }, ctx, )?), @@ -273,7 +273,7 @@ pub fn gen_py_method( PropertyType::Function { self_type, spec, - doc: spec.get_doc(meth_attrs, ctx), + doc: spec.get_doc(meth_attrs, ctx)?, }, ctx, )?), @@ -627,7 +627,7 @@ pub fn impl_py_setter_def( ) -> Result { let Ctx { pyo3_path, .. } = ctx; let python_name = property_type.null_terminated_python_name(ctx)?; - let doc = property_type.doc(ctx); + let doc = property_type.doc(ctx)?; let mut holders = Holders::new(); let setter_impl = match property_type { PropertyType::Descriptor { @@ -815,7 +815,7 @@ pub fn impl_py_getter_def( ) -> Result { let Ctx { pyo3_path, .. } = ctx; let python_name = property_type.null_terminated_python_name(ctx)?; - let doc = property_type.doc(ctx); + let doc = property_type.doc(ctx)?; let mut cfg_attrs = TokenStream::new(); if let PropertyType::Descriptor { field, .. } = &property_type { @@ -978,13 +978,17 @@ impl PropertyType<'_> { } } - fn doc(&self, ctx: &Ctx) -> Cow<'_, PythonDoc> { - match self { + fn doc(&self, ctx: &Ctx) -> Result> { + let doc = match self { PropertyType::Descriptor { field, .. } => { - Cow::Owned(utils::get_doc(&field.attrs, None, ctx)) + // FIXME: due to the clone this will not properly strip Rust documentation, maybe + // need to parse the field and doc earlier in the process? + let mut attrs = field.attrs.clone(); + Cow::Owned(utils::get_doc(&mut attrs, None, ctx)?) } PropertyType::Function { doc, .. } => Cow::Borrowed(doc), - } + }; + Ok(doc) } } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 1954e79c858..852356e06f1 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,7 +1,8 @@ -use crate::attributes::{CrateAttribute, RenamingRule}; +use crate::attributes::{self, CrateAttribute, RenamingRule}; use proc_macro2::{Span, TokenStream}; use quote::{quote, ToTokens}; use std::ffi::CString; +use syn::parse::Parse; use syn::spanned::Spanned; use syn::visit_mut::VisitMut; use syn::{punctuated::Punctuated, Token}; @@ -115,6 +116,13 @@ impl quote::ToTokens for LitCStr { #[derive(Clone)] pub struct PythonDoc(PythonDocKind); +impl PythonDoc { + /// Returns an empty docstring. + pub fn empty(ctx: &Ctx) -> Self { + PythonDoc(PythonDocKind::LitCStr(LitCStr::empty(ctx))) + } +} + #[derive(Clone)] enum PythonDocKind { LitCStr(LitCStr), @@ -123,16 +131,50 @@ enum PythonDocKind { Tokens(TokenStream), } +enum DocParseMode { + /// Currently generating docs for both Python and Rust. + Both, + /// Currently generating docs for Python only, starting from the given Span. + PythonOnly(Span), + /// Currently generating docs for Rust only, starting from the given Span. + RustOnly(Span), +} + +impl Parse for DocParseMode { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::doc_mode) { + let attribute: attributes::DocModeAttribute = input.parse()?; + let lit = attribute.value; + if lit.value() == "python" { + Ok(DocParseMode::PythonOnly(lit.span())) + } else if lit.value() == "rust" { + Ok(DocParseMode::RustOnly(lit.span())) + } else { + Err(syn::Error::new( + lit.span(), + "expected `doc_mode = \"python\"` or `doc_mode = \"rust\"", + )) + } + } else if lookahead.peek(attributes::kw::end_doc_mode) { + let _: attributes::kw::end_doc_mode = input.parse()?; + Ok(DocParseMode::Both) + } else { + Err(lookahead.error()) + } + } +} + /// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string. /// /// If this doc is for a callable, the provided `text_signature` can be passed to prepend /// this to the documentation suitable for Python to extract this into the `__text_signature__` /// attribute. pub fn get_doc( - attrs: &[syn::Attribute], + attrs: &mut Vec, mut text_signature: Option, ctx: &Ctx, -) -> PythonDoc { +) -> syn::Result { let Ctx { pyo3_path, .. } = ctx; // insert special divider between `__text_signature__` and doc // (assume text_signature is itself well-formed) @@ -144,31 +186,92 @@ pub fn get_doc( let mut first = true; let mut current_part = text_signature.unwrap_or_default(); - for attr in attrs { + let mut mode = DocParseMode::Both; + + let mut error: Option = None; + + attrs.retain(|attr| { if attr.path().is_ident("doc") { - if let Ok(nv) = attr.meta.require_name_value() { - if !first { - current_part.push('\n'); - } else { - first = false; + // if not in Rust-only mode, we process the doc attribute to add to the Python docstring + if !matches!(mode, DocParseMode::RustOnly(_)) { + if let Ok(nv) = attr.meta.require_name_value() { + if !first { + current_part.push('\n'); + } else { + first = false; + } + if let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit_str), + .. + }) = &nv.value + { + // Strip single left space from literal strings, if needed. + // e.g. `/// Hello world` expands to #[doc = " Hello world"] + let doc_line = lit_str.value(); + current_part.push_str(doc_line.strip_prefix(' ').unwrap_or(&doc_line)); + } else { + // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)] + // Reset the string buffer, write that part, and then push this macro part too. + parts.push(current_part.to_token_stream()); + current_part.clear(); + parts.push(nv.value.to_token_stream()); + } } - if let syn::Expr::Lit(syn::ExprLit { - lit: syn::Lit::Str(lit_str), - .. - }) = &nv.value - { - // Strip single left space from literal strings, if needed. - // e.g. `/// Hello world` expands to #[doc = " Hello world"] - let doc_line = lit_str.value(); - current_part.push_str(doc_line.strip_prefix(' ').unwrap_or(&doc_line)); - } else { - // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)] - // Reset the string buffer, write that part, and then push this macro part too. - parts.push(current_part.to_token_stream()); - current_part.clear(); - parts.push(nv.value.to_token_stream()); + } + // discard doc attributes if we're in PythonOnly mode + !matches!(mode, DocParseMode::PythonOnly(_)) + } else if attr.path().is_ident("pyo3_doc_mode") { + match attr.parse_args() { + Ok(new_mode) => { + mode = new_mode; } + Err(err) => match &mut error { + Some(existing_error) => existing_error.combine(err), + None => { + error = Some(err); + } + }, } + // we processed these attributes, remove them + false + } else { + true + } + }); + + // for attr in attrs { + // if attr.path().is_ident("doc") { + // if let Ok(nv) = attr.meta.require_name_value() { + // if !first { + // current_part.push('\n'); + // } else { + // first = false; + // } + // if let syn::Expr::Lit(syn::ExprLit { + // lit: syn::Lit::Str(lit_str), + // .. + // }) = &nv.value + // { + // // Strip single left space from literal strings, if needed. + // // e.g. `/// Hello world` expands to #[doc = " Hello world"] + // let doc_line = lit_str.value(); + // current_part.push_str(doc_line.strip_prefix(' ').unwrap_or(&doc_line)); + // } else { + // // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)] + // // Reset the string buffer, write that part, and then push this macro part too. + // parts.push(current_part.to_token_stream()); + // current_part.clear(); + // parts.push(nv.value.to_token_stream()); + // } + // } + // } + // } + + // if the mode has not been ended, we return an error + match mode { + DocParseMode::Both => {} + DocParseMode::PythonOnly(span) | DocParseMode::RustOnly(span) => { + return Err(err_spanned!(span => "doc_mode must be ended with `end_doc_mode`")); } } @@ -187,17 +290,17 @@ pub fn get_doc( syn::token::Comma(Span::call_site()).to_tokens(tokens); }); - PythonDoc(PythonDocKind::Tokens( + Ok(PythonDoc(PythonDocKind::Tokens( quote!(#pyo3_path::ffi::c_str!(#tokens)), - )) + ))) } else { // Just a string doc - return directly with nul terminator let docs = CString::new(current_part).unwrap(); - PythonDoc(PythonDocKind::LitCStr(LitCStr::new( + Ok(PythonDoc(PythonDocKind::LitCStr(LitCStr::new( docs, Span::call_site(), ctx, - ))) + )))) } }