Skip to content

Commit 4d4e41c

Browse files
TptTpt
authored
Introspection: emit doc comments (#5782)
* Introspection: emit doc comments Still missing is the support of enum variant and constants doc comments Notable changes: - get_doc() now returns an intermediate structure. The `to_cstr_stream` returns the expected cstr - get_doc() returns now an option when there is no doc - erroring on nul byte is delegated to `cstr!` - PyMemberDef.doc is now set to NULL if there is no doc instead of the empty string (NULL is allowed by cpython doc) - same for PyGetSetDef.doc * Support more places
1 parent c8ec67a commit 4d4e41c

File tree

28 files changed

+720
-366
lines changed

28 files changed

+720
-366
lines changed

‎newsfragments/5782.added.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Introspection: introspect doc comments and emit them in the stubs.

‎pyo3-introspection/src/introspection.rs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result<Module> {
5050
id,
5151
name,
5252
members,
53+
doc,
5354
incomplete,
5455
} = chunk
5556
{
@@ -64,6 +65,7 @@ fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result<Module> {
6465
name,
6566
members,
6667
*incomplete,
68+
doc.as_deref(),
6769
&chunks_by_id,
6870
&chunks_by_parent,
6971
&type_hint_for_annotation_id,
@@ -74,11 +76,13 @@ fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result<Module> {
7476
bail!("No module named {main_module_name} found")
7577
}
7678

79+
#[expect(clippy::too_many_arguments)]
7780
fn convert_module(
7881
id: &str,
7982
name: &str,
8083
members: &[String],
8184
mut incomplete: bool,
85+
docstring: Option<&str>,
8286
chunks_by_id: &HashMap<&str, &Chunk>,
8387
chunks_by_parent: &HashMap<&str, Vec<&Chunk>>,
8488
type_hint_for_annotation_id: &HashMap<String, Expr>,
@@ -110,6 +114,7 @@ fn convert_module(
110114
functions,
111115
attributes,
112116
incomplete,
117+
docstring: docstring.map(Into::into),
113118
})
114119
}
115120

@@ -133,12 +138,14 @@ fn convert_members<'a>(
133138
id,
134139
members,
135140
incomplete,
141+
doc,
136142
} => {
137143
modules.push(convert_module(
138144
id,
139145
name,
140146
members,
141147
*incomplete,
148+
doc.as_deref(),
142149
chunks_by_id,
143150
chunks_by_parent,
144151
type_hint_for_annotation_id,
@@ -149,12 +156,14 @@ fn convert_members<'a>(
149156
id,
150157
bases,
151158
decorators,
159+
doc,
152160
parent: _,
153161
} => classes.push(convert_class(
154162
id,
155163
name,
156164
bases,
157165
decorators,
166+
doc.as_deref(),
158167
chunks_by_id,
159168
chunks_by_parent,
160169
type_hint_for_annotation_id,
@@ -167,12 +176,14 @@ fn convert_members<'a>(
167176
decorators,
168177
is_async,
169178
returns,
179+
doc,
170180
} => functions.push(convert_function(
171181
name,
172182
arguments,
173183
decorators,
174184
returns,
175185
*is_async,
186+
doc.as_deref(),
176187
type_hint_for_annotation_id,
177188
)),
178189
Chunk::Attribute {
@@ -181,10 +192,12 @@ fn convert_members<'a>(
181192
parent: _,
182193
value,
183194
annotation,
195+
doc,
184196
} => attributes.push(convert_attribute(
185197
name,
186198
value,
187199
annotation,
200+
doc.as_deref(),
188201
type_hint_for_annotation_id,
189202
)),
190203
}
@@ -217,11 +230,13 @@ fn convert_members<'a>(
217230
Ok((modules, classes, functions, attributes))
218231
}
219232

233+
#[expect(clippy::too_many_arguments)]
220234
fn convert_class(
221235
id: &str,
222236
name: &str,
223237
bases: &[ChunkExpr],
224238
decorators: &[ChunkExpr],
239+
docstring: Option<&str>,
225240
chunks_by_id: &HashMap<&str, &Chunk>,
226241
chunks_by_parent: &HashMap<&str, Vec<&Chunk>>,
227242
type_hint_for_annotation_id: &HashMap<String, Expr>,
@@ -249,6 +264,7 @@ fn convert_class(
249264
.map(|e| convert_expr(e, type_hint_for_annotation_id))
250265
.collect(),
251266
inner_classes: nested_classes,
267+
docstring: docstring.map(Into::into),
252268
})
253269
}
254270

@@ -258,6 +274,7 @@ fn convert_function(
258274
decorators: &[ChunkExpr],
259275
returns: &Option<ChunkExpr>,
260276
is_async: bool,
277+
docstring: Option<&str>,
261278
type_hint_for_annotation_id: &HashMap<String, Expr>,
262279
) -> Function {
263280
Function {
@@ -295,6 +312,7 @@ fn convert_function(
295312
.as_ref()
296313
.map(|a| convert_expr(a, type_hint_for_annotation_id)),
297314
is_async,
315+
docstring: docstring.map(Into::into),
298316
}
299317
}
300318

@@ -332,6 +350,7 @@ fn convert_attribute(
332350
name: &str,
333351
value: &Option<ChunkExpr>,
334352
annotation: &Option<ChunkExpr>,
353+
docstring: Option<&str>,
335354
type_hint_for_annotation_id: &HashMap<String, Expr>,
336355
) -> Attribute {
337356
Attribute {
@@ -342,6 +361,7 @@ fn convert_attribute(
342361
annotation: annotation
343362
.as_ref()
344363
.map(|a| convert_expr(a, type_hint_for_annotation_id)),
364+
docstring: docstring.map(ToString::to_string),
345365
}
346366
}
347367

@@ -646,6 +666,8 @@ enum Chunk {
646666
id: String,
647667
name: String,
648668
members: Vec<String>,
669+
#[serde(default)]
670+
doc: Option<String>,
649671
incomplete: bool,
650672
},
651673
Class {
@@ -657,6 +679,8 @@ enum Chunk {
657679
decorators: Vec<ChunkExpr>,
658680
#[serde(default)]
659681
parent: Option<String>,
682+
#[serde(default)]
683+
doc: Option<String>,
660684
},
661685
Function {
662686
#[serde(default)]
@@ -671,6 +695,8 @@ enum Chunk {
671695
returns: Option<ChunkExpr>,
672696
#[serde(default, rename = "async")]
673697
is_async: bool,
698+
#[serde(default)]
699+
doc: Option<String>,
674700
},
675701
Attribute {
676702
#[serde(default)]
@@ -682,6 +708,8 @@ enum Chunk {
682708
value: Option<ChunkExpr>,
683709
#[serde(default)]
684710
annotation: Option<ChunkExpr>,
711+
#[serde(default)]
712+
doc: Option<String>,
685713
},
686714
}
687715

‎pyo3-introspection/src/model.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ pub struct Module {
66
pub functions: Vec<Function>,
77
pub attributes: Vec<Attribute>,
88
pub incomplete: bool,
9+
pub docstring: Option<String>,
910
}
1011

1112
#[derive(Debug, Eq, PartialEq, Clone, Hash)]
@@ -17,6 +18,7 @@ pub struct Class {
1718
/// decorator like 'typing.final'
1819
pub decorators: Vec<Expr>,
1920
pub inner_classes: Vec<Class>,
21+
pub docstring: Option<String>,
2022
}
2123

2224
#[derive(Debug, Eq, PartialEq, Clone, Hash)]
@@ -28,6 +30,7 @@ pub struct Function {
2830
/// return type
2931
pub returns: Option<Expr>,
3032
pub is_async: bool,
33+
pub docstring: Option<String>,
3134
}
3235

3336
#[derive(Debug, Eq, PartialEq, Clone, Hash)]
@@ -37,6 +40,7 @@ pub struct Attribute {
3740
pub value: Option<Expr>,
3841
/// Type annotation as a Python expression
3942
pub annotation: Option<Expr>,
43+
pub docstring: Option<String>,
4044
}
4145

4246
#[derive(Debug, Eq, PartialEq, Clone, Hash)]

‎pyo3-introspection/src/stubs.rs‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,18 @@ fn module_stubs(module: &Module, parents: &[&str]) -> String {
8484
attr: "Incomplete".into(),
8585
}),
8686
is_async: false,
87+
docstring: None,
8788
},
8889
&imports,
8990
None,
9091
));
9192
}
9293

93-
let mut final_elements = imports.imports;
94+
let mut final_elements = Vec::new();
95+
if let Some(docstring) = &module.docstring {
96+
final_elements.push(format!("\"\"\"\n{docstring}\n\"\"\""));
97+
}
98+
final_elements.extend(imports.imports);
9499
final_elements.extend(elements);
95100

96101
let mut output = String::new();
@@ -135,9 +140,20 @@ fn class_stubs(class: &Class, imports: &Imports) -> String {
135140
buffer.push(')');
136141
}
137142
buffer.push(':');
138-
if class.methods.is_empty() && class.attributes.is_empty() && class.inner_classes.is_empty() {
143+
if class.docstring.is_none()
144+
&& class.methods.is_empty()
145+
&& class.attributes.is_empty()
146+
&& class.inner_classes.is_empty()
147+
{
139148
buffer.push_str(" ...");
140-
return buffer;
149+
}
150+
if let Some(docstring) = &class.docstring {
151+
buffer.push_str("\n \"\"\"");
152+
for line in docstring.lines() {
153+
buffer.push_str("\n ");
154+
buffer.push_str(line);
155+
}
156+
buffer.push_str("\n \"\"\"");
141157
}
142158
for attribute in &class.attributes {
143159
// We do the indentation
@@ -214,7 +230,16 @@ fn function_stubs(function: &Function, imports: &Imports, class_name: Option<&st
214230
buffer.push_str(" -> ");
215231
imports.serialize_expr(returns, &mut buffer);
216232
}
217-
buffer.push_str(": ...");
233+
if let Some(docstring) = &function.docstring {
234+
buffer.push_str(":\n \"\"\"");
235+
for line in docstring.lines() {
236+
buffer.push_str("\n ");
237+
buffer.push_str(line);
238+
}
239+
buffer.push_str("\n \"\"\"");
240+
} else {
241+
buffer.push_str(": ...");
242+
}
218243
buffer
219244
}
220245

@@ -228,6 +253,14 @@ fn attribute_stubs(attribute: &Attribute, imports: &Imports) -> String {
228253
buffer.push_str(" = ");
229254
imports.serialize_expr(value, &mut buffer);
230255
}
256+
if let Some(docstring) = &attribute.docstring {
257+
buffer.push_str("\n\"\"\"");
258+
for line in docstring.lines() {
259+
buffer.push('\n');
260+
buffer.push_str(line);
261+
}
262+
buffer.push_str("\n\"\"\"");
263+
}
231264
buffer
232265
}
233266

@@ -635,6 +668,7 @@ mod tests {
635668
value: Constant::Str("list[str]".into()),
636669
}),
637670
is_async: false,
671+
docstring: None,
638672
};
639673
assert_eq!(
640674
"def func(posonly, /, arg, *varargs, karg: \"str\", **kwarg: \"str\") -> \"list[str]\": ...",
@@ -676,6 +710,7 @@ mod tests {
676710
},
677711
returns: None,
678712
is_async: false,
713+
docstring: None,
679714
};
680715
assert_eq!(
681716
"def afunc(posonly=1, /, arg=True, *, karg: \"str\" = \"foo\"): ...",
@@ -697,6 +732,7 @@ mod tests {
697732
},
698733
returns: None,
699734
is_async: true,
735+
docstring: None,
700736
};
701737
assert_eq!(
702738
"async def foo(): ...",
@@ -767,6 +803,7 @@ mod tests {
767803
attr: "final".into(),
768804
}],
769805
inner_classes: Vec::new(),
806+
docstring: None,
770807
},
771808
Class {
772809
name: "int".into(),
@@ -775,6 +812,7 @@ mod tests {
775812
attributes: Vec::new(),
776813
decorators: Vec::new(),
777814
inner_classes: Vec::new(),
815+
docstring: None,
778816
},
779817
],
780818
functions: vec![Function {
@@ -789,9 +827,11 @@ mod tests {
789827
},
790828
returns: Some(big_type.clone()),
791829
is_async: false,
830+
docstring: None,
792831
}],
793832
attributes: Vec::new(),
794833
incomplete: true,
834+
docstring: None,
795835
},
796836
&["foo"],
797837
);

0 commit comments

Comments
 (0)