From 26b5b29ceffa4ce1df07f069e09159acd30a460e Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 31 Oct 2024 11:02:19 +0800 Subject: [PATCH 01/17] don't append `None` in annotation, lambda and typealias --- Python/codegen.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 624d4f7ce14f21..4beeac150bb724 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -672,9 +672,7 @@ codegen_setup_annotations_scope(compiler *c, location loc, codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, key, loc.lineno, NULL, &umd)); - // Insert None into consts to prevent an annotation - // appearing to be a docstring - _PyCompile_AddConst(c, Py_None); + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); // if .format != 1: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); @@ -1600,9 +1598,8 @@ codegen_typealias_body(compiler *c, stmt_ty s) ADDOP_LOAD_CONST_NEW(c, loc, defaults); RETURN_IF_ERROR( codegen_setup_annotations_scope(c, LOC(s), s, name)); - /* Make None the first constant, so the evaluate function can't have a - docstring. */ - RETURN_IF_ERROR(_PyCompile_AddConst(c, Py_None)); + + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); VISIT_IN_SCOPE(c, expr, s->v.TypeAlias.value); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -1898,9 +1895,7 @@ codegen_lambda(compiler *c, expr_ty e) codegen_enter_scope(c, &_Py_STR(anon_lambda), COMPILE_SCOPE_LAMBDA, (void *)e, e->lineno, NULL, &umd)); - /* Make None the first constant, so the lambda can't have a - docstring. */ - RETURN_IF_ERROR(_PyCompile_AddConst(c, Py_None)); + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); if (SYMTABLE_ENTRY(c)->ste_generator) { From a366331600f6fdf844d5ff796691aea0a6d296d2 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 31 Oct 2024 15:54:12 +0800 Subject: [PATCH 02/17] set `ste_has_docstring` for class and module --- Python/codegen.c | 3 ++- Python/symtable.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Python/codegen.c b/Python/codegen.c index 4beeac150bb724..08fd1d704debbf 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1239,10 +1239,12 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags RETURN_IF_ERROR( codegen_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL, &umd)); + PySTEntryObject *ste = SYMTABLE_ENTRY(c); Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); if (docstring) { + assert(ste->ste_has_docstring); first_instr = 1; docstring = _PyCompile_CleanDoc(docstring); if (docstring == NULL) { @@ -1256,7 +1258,6 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags NEW_JUMP_TARGET_LABEL(c, start); USE_LABEL(c, start); - PySTEntryObject *ste = SYMTABLE_ENTRY(c); bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; if (add_stopiteration_handler) { /* codegen_wrap_in_stopiteration_handler will push a block, so we need to account for that */ diff --git a/Python/symtable.c b/Python/symtable.c index 32d715197c541b..7a9890f5df1b10 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -434,6 +434,12 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; + if (_PyAST_GetDocString(seq)) { + PySTEntryObject *mod_ste = _PySymtable_Lookup(st, mod); + assert(mod_ste); + mod_ste->ste_has_docstring = 1; + Py_DECREF(mod_ste); + } for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, (stmt_ty)asdl_seq_GET(seq, i))) @@ -1909,6 +1915,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } } + + if (_PyAST_GetDocString(s->v.ClassDef.body)) { + PySTEntryObject *class_ste = _PySymtable_Lookup(st, s); + assert(class_ste); + class_ste->ste_has_docstring = 1; + Py_DECREF(class_ste); + } + VISIT_SEQ(st, stmt, s->v.ClassDef.body); if (!symtable_exit_block(st)) return 0; From 356025fe1622e341de042be147ce946c263e73d2 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 31 Oct 2024 16:27:33 +0800 Subject: [PATCH 03/17] fix lambda test --- Lib/test/test_compile.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 519a1207afb1fc..e065ecb5423185 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -342,6 +342,10 @@ def test_lambda_doc(self): l = lambda: "foo" self.assertIsNone(l.__doc__) + def test_lambda_consts(self): + l = lambda: "this is the only const" + self.assertEqual(len(l.__code__.co_consts), 1) + def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') @@ -790,10 +794,10 @@ def check_same_constant(const): # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} - self.assertIs(f1.__code__.co_consts[1], - f2.__code__.co_consts[1][0]) - self.assertIs(next(iter(f3.__code__.co_consts[1])), - f2.__code__.co_consts[1]) + self.assertIs(f1.__code__.co_consts[0], + f2.__code__.co_consts[0][0]) + self.assertIs(next(iter(f3.__code__.co_consts[0])), + f2.__code__.co_consts[0]) # {0} is converted to a constant frozenset({0}) by the peephole # optimizer From 363536ebd24b968f0c90ae1c52425aec0f6ab327 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 31 Oct 2024 19:02:46 +0800 Subject: [PATCH 04/17] add class docstring test --- Lib/test/test_code.py | 13 +++++++++++++ Lib/test/test_compile.py | 1 + 2 files changed, 14 insertions(+) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 93c65a82508dcb..2737fec295c6eb 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -178,6 +178,19 @@ nlocals: 3 flags: 3 consts: ("'hello'", "'world'") + +>>> class class_with_docstring: +... '''This is a docstring for class''' +... pass + +>>> print(class_with_docstring.__doc__) +This is a docstring for class + +>>> class class_without_docstring: +... pass + +>>> print(class_without_docstring.__doc__) +None """ import copy diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index e065ecb5423185..e398b4a76a5db1 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -345,6 +345,7 @@ def test_lambda_doc(self): def test_lambda_consts(self): l = lambda: "this is the only const" self.assertEqual(len(l.__code__.co_consts), 1) + self.assertEqual(l.__code__.co_consts[0], "this is the only const") def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' From d8d488fe3742fc4fe1ab9c974ed93ba6144afb0c Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 31 Oct 2024 21:51:11 +0800 Subject: [PATCH 05/17] add news and ack entry --- Lib/test/test_code.py | 1 + Misc/ACKS | 1 + .../2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2737fec295c6eb..99dabab915923e 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -181,6 +181,7 @@ >>> class class_with_docstring: ... '''This is a docstring for class''' +... '''This line is not docstring''' ... pass >>> print(class_with_docstring.__doc__) diff --git a/Misc/ACKS b/Misc/ACKS index d03c70f6db87bf..1a25088052f4e1 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -820,6 +820,7 @@ Tomáš Hrnčiar Miro Hrončok Chiu-Hsiang Hsu Chih-Hao Huang +Xuanteng Huang Christian Hudon Benoît Hudson Lawrence Hudson diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst new file mode 100644 index 00000000000000..bea732d45f87de --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst @@ -0,0 +1,2 @@ +Set ``ste_has_docstring`` for ``class`` and ``module`` in corresponding +symbol table entry. From a67a4ea4c0956bef4311b7d7edadb0ff4fdbdc64 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 1 Nov 2024 09:41:45 +0800 Subject: [PATCH 06/17] add ste fetch and news --- .../2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst | 4 ++-- Python/symtable.c | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst index bea732d45f87de..2464ac78cf429b 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst @@ -1,2 +1,2 @@ -Set ``ste_has_docstring`` for ``class`` and ``module`` in corresponding -symbol table entry. +Following :gh:`126101`, for :ref:`codeobjects` like lambda, annotation and type alias, +we no longer add ``None`` to its :attr:`~codeobject.co_consts`. diff --git a/Python/symtable.c b/Python/symtable.c index 7a9890f5df1b10..638d966c2b7723 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -435,10 +435,8 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) case Module_kind: seq = mod->v.Module.body; if (_PyAST_GetDocString(seq)) { - PySTEntryObject *mod_ste = _PySymtable_Lookup(st, mod); - assert(mod_ste); - mod_ste->ste_has_docstring = 1; - Py_DECREF(mod_ste); + assert(st->st_cur); + st->st_cur->ste_has_docstring = 1; } for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, @@ -1917,10 +1915,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } if (_PyAST_GetDocString(s->v.ClassDef.body)) { - PySTEntryObject *class_ste = _PySymtable_Lookup(st, s); - assert(class_ste); - class_ste->ste_has_docstring = 1; - Py_DECREF(class_ste); + assert(st->st_cur); + st->st_cur->ste_has_docstring = 1; } VISIT_SEQ(st, stmt, s->v.ClassDef.body); From 89426760ec6ce6dd631285d6caff45442915c1df Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Sun, 3 Nov 2024 21:58:00 +0800 Subject: [PATCH 07/17] simplify code --- Lib/test/test_compile.py | 3 +-- Python/codegen.c | 4 ++-- Python/symtable.c | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index e398b4a76a5db1..3ef7790391f58f 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -344,8 +344,7 @@ def test_lambda_doc(self): def test_lambda_consts(self): l = lambda: "this is the only const" - self.assertEqual(len(l.__code__.co_consts), 1) - self.assertEqual(l.__code__.co_consts[0], "this is the only const") + self.assertEqual(l.__code__.co_consts, ("this is the only const",)) def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' diff --git a/Python/codegen.c b/Python/codegen.c index 08fd1d704debbf..48942d0a71f6ab 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1243,8 +1243,8 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); - if (docstring) { - assert(ste->ste_has_docstring); + if (ste->ste_has_docstring) { + assert(docstring); first_instr = 1; docstring = _PyCompile_CleanDoc(docstring); if (docstring == NULL) { diff --git a/Python/symtable.c b/Python/symtable.c index 638d966c2b7723..ebddb0b93fca0a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -435,7 +435,6 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) case Module_kind: seq = mod->v.Module.body; if (_PyAST_GetDocString(seq)) { - assert(st->st_cur); st->st_cur->ste_has_docstring = 1; } for (i = 0; i < asdl_seq_LEN(seq); i++) @@ -1915,7 +1914,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } if (_PyAST_GetDocString(s->v.ClassDef.body)) { - assert(st->st_cur); st->st_cur->ste_has_docstring = 1; } From fe1db96db3b773a5f966f45a6d1f75c7ff4bcdcc Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Mon, 4 Nov 2024 10:41:49 +0800 Subject: [PATCH 08/17] add docstring test under `-OO` --- Lib/test/test_code.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 99dabab915923e..2a1b26e8a1ffd1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -868,6 +868,33 @@ def f(): 3 * [(42, 42, None, None)], ) + @cpython_only + def test_docstring_under_o2(self): + code = textwrap.dedent(''' + def has_docstring(x, y): + """This is a first-line doc string""" + """This is a second-line doc string""" + a = x + y + b = x - y + return a, b + + + def no_docstring(x): + def g(y): + return x + y + return g + + + async def async_func(): + """asynf function doc string""" + pass + + + for func in [has_docstring, no_docstring(4), async_func]: + assert(func.__doc__ is None) + ''') + + rc, out, err = assert_python_ok('-OO', '-c', code) if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi From 82a32c8c441836a8d817fb316264985630b95017 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Mon, 4 Nov 2024 21:59:12 +0800 Subject: [PATCH 09/17] simplify astfold_body --- Python/ast_opt.c | 38 --------------------------- Python/codegen.c | 66 ++++++++++++++++++++++++++++++----------------- Python/symtable.c | 8 +++--- 3 files changed, 47 insertions(+), 65 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 01e208b88eca8b..7897cb087c11a5 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -673,48 +673,10 @@ static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimize } \ } - -static int -stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) -{ - if (idx >= asdl_seq_LEN(stmts)) { - return 0; - } - for (Py_ssize_t i = idx; i < asdl_seq_LEN(stmts) - 1; i++) { - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, i+1); - asdl_seq_SET(stmts, i, st); - } - stmts->size--; - return 1; -} - static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTOptimizeState *state) { - int docstring = _PyAST_GetDocString(stmts) != NULL; - if (docstring && (state->optimize >= 2)) { - /* remove the docstring */ - if (!stmt_seq_remove_item(stmts, 0)) { - return 0; - } - docstring = 0; - } CALL_SEQ(astfold_stmt, stmt, stmts); - if (!docstring && _PyAST_GetDocString(stmts) != NULL) { - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - asdl_expr_seq *values = _Py_asdl_expr_seq_new(1, ctx_); - if (!values) { - return 0; - } - asdl_seq_SET(values, 0, st->v.Expr.value); - expr_ty expr = _PyAST_JoinedStr(values, st->lineno, st->col_offset, - st->end_lineno, st->end_col_offset, - ctx_); - if (!expr) { - return 0; - } - st->v.Expr.value = expr; - } return 1; } diff --git a/Python/codegen.c b/Python/codegen.c index 48942d0a71f6ab..bd9fff7274a5c0 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -759,6 +759,14 @@ _PyCodegen_Expression(compiler *c, expr_ty e) return SUCCESS; } +int codegen_is_stmt_docstring(stmt_ty st) { + if (st->kind != Expr_kind) { + return 0; + } + expr_ty e = st->v.Expr.value; + return (e->kind == Constant_kind) && PyUnicode_CheckExact(e->v.Constant.value); +} + /* Compile a sequence of statements, checking for a docstring and for annotations. */ @@ -778,19 +786,25 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */ PyObject *docstring = _PyAST_GetDocString(stmts); if (docstring) { - first_instr = 1; - /* set docstring */ - assert(OPTIMIZATION_LEVEL(c) < 2); - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; + if (SYMTABLE_ENTRY(c)->ste_has_docstring) { + assert(OPTIMIZATION_LEVEL(c) < 2); + first_instr = 1; + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); + } + // Skip string literals + while ((first_instr < asdl_seq_LEN(stmts)) && + codegen_is_stmt_docstring(asdl_seq_GET(stmts, first_instr))) { + first_instr++; } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { @@ -1242,18 +1256,24 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags PySTEntryObject *ste = SYMTABLE_ENTRY(c); Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); - assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); - if (ste->ste_has_docstring) { - assert(docstring); - first_instr = 1; - docstring = _PyCompile_CleanDoc(docstring); - if (docstring == NULL) { - _PyCompile_ExitScope(c); - return ERROR; + if (docstring) { + if (ste->ste_has_docstring) { + assert(OPTIMIZATION_LEVEL(c) < 2); + first_instr = 1; + docstring = _PyCompile_CleanDoc(docstring); + if (docstring == NULL) { + _PyCompile_ExitScope(c); + return ERROR; + } + Py_ssize_t idx = _PyCompile_AddConst(c, docstring); + Py_DECREF(docstring); + RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); + } + // Skip string literals + while ((first_instr < asdl_seq_LEN(body)) && + codegen_is_stmt_docstring(asdl_seq_GET(body, first_instr))) { + first_instr++; } - Py_ssize_t idx = _PyCompile_AddConst(c, docstring); - Py_DECREF(docstring); - RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); } NEW_JUMP_TARGET_LABEL(c, start); diff --git a/Python/symtable.c b/Python/symtable.c index ebddb0b93fca0a..d60461a4bb1175 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -434,7 +434,7 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; - if (_PyAST_GetDocString(seq)) { + if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(seq)) { st->st_cur->ste_has_docstring = 1; } for (i = 0; i < asdl_seq_LEN(seq); i++) @@ -1846,7 +1846,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } - if (_PyAST_GetDocString(s->v.FunctionDef.body)) { + if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(s->v.FunctionDef.body)) { new_ste->ste_has_docstring = 1; } @@ -1913,7 +1913,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } } - if (_PyAST_GetDocString(s->v.ClassDef.body)) { + if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(s->v.ClassDef.body)) { st->st_cur->ste_has_docstring = 1; } @@ -2182,7 +2182,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } - if (_PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { + if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { new_ste->ste_has_docstring = 1; } From 1b1fcae37f682b4ff63089484b5654dc34968e23 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Mon, 4 Nov 2024 22:51:07 +0800 Subject: [PATCH 10/17] use `optimize` when generate symtable --- Include/internal/pycore_symtable.h | 3 +- Python/compile.c | 2 +- Python/symtable.c | 87 +++++++++++++++++------------- 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 91dac767d5885b..226cda59b6954f 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -141,7 +141,8 @@ extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, - _PyFutureFeatures *future); + _PyFutureFeatures *future, + int optimize); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); diff --git a/Python/compile.c b/Python/compile.c index cbfba7f493e07d..f054706a551c16 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -129,7 +129,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyAST_Optimize(mod, arena, c->c_optimize, merged)) { return ERROR; } - c->c_st = _PySymtable_Build(mod, filename, &c->c_future); + c->c_st = _PySymtable_Build(mod, filename, &c->c_future, c->c_optimize); if (c->c_st == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_SystemError, "no symtable"); diff --git a/Python/symtable.c b/Python/symtable.c index d60461a4bb1175..b81d9a106528f7 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -236,7 +236,7 @@ static int symtable_analyze(struct symtable *st); static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, _Py_SourceLocation loc); static int symtable_exit_block(struct symtable *st); -static int symtable_visit_stmt(struct symtable *st, stmt_ty s); +static int symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_type_param(struct symtable *st, type_param_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); @@ -244,7 +244,7 @@ static int symtable_visit_listcomp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty s); static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); -static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); +static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty, int optimize); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); @@ -255,7 +255,7 @@ static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); -static int symtable_visit_match_case(struct symtable *st, match_case_ty m); +static int symtable_visit_match_case(struct symtable *st, match_case_ty m, int optimize); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc); @@ -393,7 +393,7 @@ symtable_new(void) } struct symtable * -_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) +_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future, int optimize) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; @@ -434,12 +434,12 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; - if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(seq)) { + if ((optimize < 2) && _PyAST_GetDocString(seq)) { st->st_cur->ste_has_docstring = 1; } for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, - (stmt_ty)asdl_seq_GET(seq, i))) + (stmt_ty)asdl_seq_GET(seq, i), optimize)) goto error; break; case Expression_kind: @@ -450,7 +450,7 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) seq = mod->v.Interactive.body; for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, - (stmt_ty)asdl_seq_GET(seq, i))) + (stmt_ty)asdl_seq_GET(seq, i), optimize)) goto error; break; case FunctionType_kind: @@ -1706,6 +1706,17 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, } \ } while(0) +#define VISIT_SEQ_OPT(ST, TYPE, SEQ, OPT) \ + do { \ + Py_ssize_t i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt, (OPT))) \ + return 0; \ + } \ + } while(0) + #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) \ do { \ Py_ssize_t i; \ @@ -1814,7 +1825,7 @@ maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s) } static int -symtable_visit_stmt(struct symtable *st, stmt_ty s) +symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) { ENTER_RECURSIVE(st); switch (s->kind) { @@ -1846,7 +1857,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } - if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(s->v.FunctionDef.body)) { + if ((optimize < 2) && _PyAST_GetDocString(s->v.FunctionDef.body)) { new_ste->ste_has_docstring = 1; } @@ -1861,7 +1872,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); - VISIT_SEQ(st, stmt, s->v.FunctionDef.body); + VISIT_SEQ_OPT(st, stmt, s->v.FunctionDef.body, optimize); if (!symtable_exit_block(st)) return 0; if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { @@ -1913,11 +1924,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } } - if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(s->v.ClassDef.body)) { + if ((optimize < 2) && _PyAST_GetDocString(s->v.ClassDef.body)) { st->st_cur->ste_has_docstring = 1; } - VISIT_SEQ(st, stmt, s->v.ClassDef.body); + VISIT_SEQ_OPT(st, stmt, s->v.ClassDef.body, optimize); if (!symtable_exit_block(st)) return 0; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { @@ -2022,26 +2033,26 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); - VISIT_SEQ(st, stmt, s->v.For.body); + VISIT_SEQ_OPT(st, stmt, s->v.For.body, optimize); if (s->v.For.orelse) - VISIT_SEQ(st, stmt, s->v.For.orelse); + VISIT_SEQ_OPT(st, stmt, s->v.For.orelse, optimize); break; case While_kind: VISIT(st, expr, s->v.While.test); - VISIT_SEQ(st, stmt, s->v.While.body); + VISIT_SEQ_OPT(st, stmt, s->v.While.body, optimize); if (s->v.While.orelse) - VISIT_SEQ(st, stmt, s->v.While.orelse); + VISIT_SEQ_OPT(st, stmt, s->v.While.orelse, optimize); break; case If_kind: /* XXX if 0: and lookup_yield() hacks */ VISIT(st, expr, s->v.If.test); - VISIT_SEQ(st, stmt, s->v.If.body); + VISIT_SEQ_OPT(st, stmt, s->v.If.body, optimize); if (s->v.If.orelse) - VISIT_SEQ(st, stmt, s->v.If.orelse); + VISIT_SEQ_OPT(st, stmt, s->v.If.orelse, optimize); break; case Match_kind: VISIT(st, expr, s->v.Match.subject); - VISIT_SEQ(st, match_case, s->v.Match.cases); + VISIT_SEQ_OPT(st, match_case, s->v.Match.cases, optimize); break; case Raise_kind: if (s->v.Raise.exc) { @@ -2052,16 +2063,16 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } break; case Try_kind: - VISIT_SEQ(st, stmt, s->v.Try.body); - VISIT_SEQ(st, excepthandler, s->v.Try.handlers); - VISIT_SEQ(st, stmt, s->v.Try.orelse); - VISIT_SEQ(st, stmt, s->v.Try.finalbody); + VISIT_SEQ_OPT(st, stmt, s->v.Try.body, optimize); + VISIT_SEQ_OPT(st, excepthandler, s->v.Try.handlers, optimize); + VISIT_SEQ_OPT(st, stmt, s->v.Try.orelse, optimize); + VISIT_SEQ_OPT(st, stmt, s->v.Try.finalbody, optimize); break; case TryStar_kind: - VISIT_SEQ(st, stmt, s->v.TryStar.body); - VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); - VISIT_SEQ(st, stmt, s->v.TryStar.orelse); - VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); + VISIT_SEQ_OPT(st, stmt, s->v.TryStar.body, optimize); + VISIT_SEQ_OPT(st, excepthandler, s->v.TryStar.handlers, optimize); + VISIT_SEQ_OPT(st, stmt, s->v.TryStar.orelse, optimize); + VISIT_SEQ_OPT(st, stmt, s->v.TryStar.finalbody, optimize); break; case Assert_kind: VISIT(st, expr, s->v.Assert.test); @@ -2151,7 +2162,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case With_kind: VISIT_SEQ(st, withitem, s->v.With.items); - VISIT_SEQ(st, stmt, s->v.With.body); + VISIT_SEQ_OPT(st, stmt, s->v.With.body, optimize); break; case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) @@ -2182,7 +2193,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } - if ((_Py_GetConfig()->optimization_level < 2) && _PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { + if ((optimize < 2) && _PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { new_ste->ste_has_docstring = 1; } @@ -2199,7 +2210,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); - VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); + VISIT_SEQ_OPT(st, stmt, s->v.AsyncFunctionDef.body, optimize); if (!symtable_exit_block(st)) return 0; if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { @@ -2214,7 +2225,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } VISIT_SEQ(st, withitem, s->v.AsyncWith.items); - VISIT_SEQ(st, stmt, s->v.AsyncWith.body); + VISIT_SEQ_OPT(st, stmt, s->v.AsyncWith.body, optimize); break; case AsyncFor_kind: maybe_set_ste_coroutine_for_module(st, s); @@ -2223,9 +2234,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT(st, expr, s->v.AsyncFor.target); VISIT(st, expr, s->v.AsyncFor.iter); - VISIT_SEQ(st, stmt, s->v.AsyncFor.body); + VISIT_SEQ_OPT(st, stmt, s->v.AsyncFor.body, optimize); if (s->v.AsyncFor.orelse) - VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse); + VISIT_SEQ_OPT(st, stmt, s->v.AsyncFor.orelse, optimize); break; } LEAVE_RECURSIVE(st); @@ -2808,14 +2819,14 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a) static int -symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) +symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh, int optimize) { if (eh->v.ExceptHandler.type) VISIT(st, expr, eh->v.ExceptHandler.type); if (eh->v.ExceptHandler.name) if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL, LOCATION(eh))) return 0; - VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body); + VISIT_SEQ_OPT(st, stmt, eh->v.ExceptHandler.body, optimize); return 1; } @@ -2830,13 +2841,13 @@ symtable_visit_withitem(struct symtable *st, withitem_ty item) } static int -symtable_visit_match_case(struct symtable *st, match_case_ty m) +symtable_visit_match_case(struct symtable *st, match_case_ty m, int optimize) { VISIT(st, pattern, m->pattern); if (m->guard) { VISIT(st, expr, m->guard); } - VISIT_SEQ(st, stmt, m->body); + VISIT_SEQ_OPT(st, stmt, m->body, optimize); return 1; } @@ -3079,7 +3090,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, return NULL; } future.ff_features |= flags->cf_flags; - st = _PySymtable_Build(mod, filename, &future); + st = _PySymtable_Build(mod, filename, &future, _Py_GetConfig()->optimization_level); _PyArena_Free(arena); return st; } From 88f6acf9b92979345069da8c9fa826624e5f9aa6 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 15:36:03 +0800 Subject: [PATCH 11/17] add const string multiply test --- Lib/test/test_compile.py | 4 ++++ Python/ast_opt.c | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 3ef7790391f58f..f7ea923ef17672 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -906,6 +906,9 @@ def with_fstring(): def with_const_expression(): "also" + " not docstring" + + def multiple_const_strings(): + "not docstring " * 3 """) for opt in [0, 1, 2]: @@ -922,6 +925,7 @@ def with_const_expression(): self.assertIsNone(ns['two_strings'].__doc__) self.assertIsNone(ns['with_fstring'].__doc__) self.assertIsNone(ns['with_const_expression'].__doc__) + self.assertIsNone(ns['multiple_const_strings'].__doc__) @support.cpython_only def test_docstring_interactive_mode(self): diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 7897cb087c11a5..6ccc41e6ee6c4f 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -676,7 +676,23 @@ static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimize static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTOptimizeState *state) { + int docstring = _PyAST_GetDocString(stmts) != NULL; CALL_SEQ(astfold_stmt, stmt, stmts); + if (!docstring && _PyAST_GetDocString(stmts) != NULL) { + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + asdl_expr_seq *values = _Py_asdl_expr_seq_new(1, ctx_); + if (!values) { + return 0; + } + asdl_seq_SET(values, 0, st->v.Expr.value); + expr_ty expr = _PyAST_JoinedStr(values, st->lineno, st->col_offset, + st->end_lineno, st->end_col_offset, + ctx_); + if (!expr) { + return 0; + } + st->v.Expr.value = expr; + } return 1; } From 119db3bd5a7d318d63eb5e988850875b457ccbf4 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 15:42:18 +0800 Subject: [PATCH 12/17] hide private function --- Python/codegen.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/codegen.c b/Python/codegen.c index bd9fff7274a5c0..4b4d0219811a82 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -759,7 +759,8 @@ _PyCodegen_Expression(compiler *c, expr_ty e) return SUCCESS; } -int codegen_is_stmt_docstring(stmt_ty st) { +static int +codegen_is_stmt_docstring(stmt_ty st) { if (st->kind != Expr_kind) { return 0; } From 24e2c8f219a4f712a561a2b6bc2b532f8d003f32 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 16:20:39 +0800 Subject: [PATCH 13/17] update docs --- Doc/library/ast.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 22d8c87cb58e78..ab2a32cc8aa606 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2184,6 +2184,10 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionchanged:: 3.14 + :func:`ast.parse` now will **NOT** remove the docstring in nodes + when ``optimize=2``. + .. function:: unparse(ast_obj) From b81f4a4225887b3a4820929c0d5a56a4ac88a5d8 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 21:19:56 +0800 Subject: [PATCH 14/17] revert ast_opt hacking commits --- Doc/library/ast.rst | 4 -- Include/internal/pycore_symtable.h | 3 +- Python/ast_opt.c | 22 ++++++++ Python/codegen.c | 67 ++++++++--------------- Python/compile.c | 2 +- Python/symtable.c | 87 +++++++++++++----------------- 6 files changed, 85 insertions(+), 100 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ab2a32cc8aa606..22d8c87cb58e78 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2184,10 +2184,6 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. - .. versionchanged:: 3.14 - :func:`ast.parse` now will **NOT** remove the docstring in nodes - when ``optimize=2``. - .. function:: unparse(ast_obj) diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 226cda59b6954f..91dac767d5885b 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -141,8 +141,7 @@ extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, - _PyFutureFeatures *future, - int optimize); + _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 6ccc41e6ee6c4f..01e208b88eca8b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -673,10 +673,32 @@ static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimize } \ } + +static int +stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) +{ + if (idx >= asdl_seq_LEN(stmts)) { + return 0; + } + for (Py_ssize_t i = idx; i < asdl_seq_LEN(stmts) - 1; i++) { + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, i+1); + asdl_seq_SET(stmts, i, st); + } + stmts->size--; + return 1; +} + static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTOptimizeState *state) { int docstring = _PyAST_GetDocString(stmts) != NULL; + if (docstring && (state->optimize >= 2)) { + /* remove the docstring */ + if (!stmt_seq_remove_item(stmts, 0)) { + return 0; + } + docstring = 0; + } CALL_SEQ(astfold_stmt, stmt, stmts); if (!docstring && _PyAST_GetDocString(stmts) != NULL) { stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); diff --git a/Python/codegen.c b/Python/codegen.c index 4b4d0219811a82..48942d0a71f6ab 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -759,15 +759,6 @@ _PyCodegen_Expression(compiler *c, expr_ty e) return SUCCESS; } -static int -codegen_is_stmt_docstring(stmt_ty st) { - if (st->kind != Expr_kind) { - return 0; - } - expr_ty e = st->v.Expr.value; - return (e->kind == Constant_kind) && PyUnicode_CheckExact(e->v.Constant.value); -} - /* Compile a sequence of statements, checking for a docstring and for annotations. */ @@ -787,25 +778,19 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */ PyObject *docstring = _PyAST_GetDocString(stmts); if (docstring) { - if (SYMTABLE_ENTRY(c)->ste_has_docstring) { - assert(OPTIMIZATION_LEVEL(c) < 2); - first_instr = 1; - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; - } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); - } - // Skip string literals - while ((first_instr < asdl_seq_LEN(stmts)) && - codegen_is_stmt_docstring(asdl_seq_GET(stmts, first_instr))) { - first_instr++; + first_instr = 1; + /* set docstring */ + assert(OPTIMIZATION_LEVEL(c) < 2); + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { @@ -1257,24 +1242,18 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags PySTEntryObject *ste = SYMTABLE_ENTRY(c); Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); - if (docstring) { - if (ste->ste_has_docstring) { - assert(OPTIMIZATION_LEVEL(c) < 2); - first_instr = 1; - docstring = _PyCompile_CleanDoc(docstring); - if (docstring == NULL) { - _PyCompile_ExitScope(c); - return ERROR; - } - Py_ssize_t idx = _PyCompile_AddConst(c, docstring); - Py_DECREF(docstring); - RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); - } - // Skip string literals - while ((first_instr < asdl_seq_LEN(body)) && - codegen_is_stmt_docstring(asdl_seq_GET(body, first_instr))) { - first_instr++; + assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); + if (ste->ste_has_docstring) { + assert(docstring); + first_instr = 1; + docstring = _PyCompile_CleanDoc(docstring); + if (docstring == NULL) { + _PyCompile_ExitScope(c); + return ERROR; } + Py_ssize_t idx = _PyCompile_AddConst(c, docstring); + Py_DECREF(docstring); + RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); } NEW_JUMP_TARGET_LABEL(c, start); diff --git a/Python/compile.c b/Python/compile.c index f054706a551c16..cbfba7f493e07d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -129,7 +129,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyAST_Optimize(mod, arena, c->c_optimize, merged)) { return ERROR; } - c->c_st = _PySymtable_Build(mod, filename, &c->c_future, c->c_optimize); + c->c_st = _PySymtable_Build(mod, filename, &c->c_future); if (c->c_st == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_SystemError, "no symtable"); diff --git a/Python/symtable.c b/Python/symtable.c index b81d9a106528f7..ebddb0b93fca0a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -236,7 +236,7 @@ static int symtable_analyze(struct symtable *st); static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, _Py_SourceLocation loc); static int symtable_exit_block(struct symtable *st); -static int symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize); +static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_type_param(struct symtable *st, type_param_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); @@ -244,7 +244,7 @@ static int symtable_visit_listcomp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty s); static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); -static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty, int optimize); +static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); @@ -255,7 +255,7 @@ static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); -static int symtable_visit_match_case(struct symtable *st, match_case_ty m, int optimize); +static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc); @@ -393,7 +393,7 @@ symtable_new(void) } struct symtable * -_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future, int optimize) +_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; @@ -434,12 +434,12 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future, int switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; - if ((optimize < 2) && _PyAST_GetDocString(seq)) { + if (_PyAST_GetDocString(seq)) { st->st_cur->ste_has_docstring = 1; } for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, - (stmt_ty)asdl_seq_GET(seq, i), optimize)) + (stmt_ty)asdl_seq_GET(seq, i))) goto error; break; case Expression_kind: @@ -450,7 +450,7 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future, int seq = mod->v.Interactive.body; for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, - (stmt_ty)asdl_seq_GET(seq, i), optimize)) + (stmt_ty)asdl_seq_GET(seq, i))) goto error; break; case FunctionType_kind: @@ -1706,17 +1706,6 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, } \ } while(0) -#define VISIT_SEQ_OPT(ST, TYPE, SEQ, OPT) \ - do { \ - Py_ssize_t i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt, (OPT))) \ - return 0; \ - } \ - } while(0) - #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) \ do { \ Py_ssize_t i; \ @@ -1825,7 +1814,7 @@ maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s) } static int -symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) +symtable_visit_stmt(struct symtable *st, stmt_ty s) { ENTER_RECURSIVE(st); switch (s->kind) { @@ -1857,7 +1846,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) return 0; } - if ((optimize < 2) && _PyAST_GetDocString(s->v.FunctionDef.body)) { + if (_PyAST_GetDocString(s->v.FunctionDef.body)) { new_ste->ste_has_docstring = 1; } @@ -1872,7 +1861,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) } Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); - VISIT_SEQ_OPT(st, stmt, s->v.FunctionDef.body, optimize); + VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) return 0; if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { @@ -1924,11 +1913,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) } } - if ((optimize < 2) && _PyAST_GetDocString(s->v.ClassDef.body)) { + if (_PyAST_GetDocString(s->v.ClassDef.body)) { st->st_cur->ste_has_docstring = 1; } - VISIT_SEQ_OPT(st, stmt, s->v.ClassDef.body, optimize); + VISIT_SEQ(st, stmt, s->v.ClassDef.body); if (!symtable_exit_block(st)) return 0; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { @@ -2033,26 +2022,26 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); - VISIT_SEQ_OPT(st, stmt, s->v.For.body, optimize); + VISIT_SEQ(st, stmt, s->v.For.body); if (s->v.For.orelse) - VISIT_SEQ_OPT(st, stmt, s->v.For.orelse, optimize); + VISIT_SEQ(st, stmt, s->v.For.orelse); break; case While_kind: VISIT(st, expr, s->v.While.test); - VISIT_SEQ_OPT(st, stmt, s->v.While.body, optimize); + VISIT_SEQ(st, stmt, s->v.While.body); if (s->v.While.orelse) - VISIT_SEQ_OPT(st, stmt, s->v.While.orelse, optimize); + VISIT_SEQ(st, stmt, s->v.While.orelse); break; case If_kind: /* XXX if 0: and lookup_yield() hacks */ VISIT(st, expr, s->v.If.test); - VISIT_SEQ_OPT(st, stmt, s->v.If.body, optimize); + VISIT_SEQ(st, stmt, s->v.If.body); if (s->v.If.orelse) - VISIT_SEQ_OPT(st, stmt, s->v.If.orelse, optimize); + VISIT_SEQ(st, stmt, s->v.If.orelse); break; case Match_kind: VISIT(st, expr, s->v.Match.subject); - VISIT_SEQ_OPT(st, match_case, s->v.Match.cases, optimize); + VISIT_SEQ(st, match_case, s->v.Match.cases); break; case Raise_kind: if (s->v.Raise.exc) { @@ -2063,16 +2052,16 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) } break; case Try_kind: - VISIT_SEQ_OPT(st, stmt, s->v.Try.body, optimize); - VISIT_SEQ_OPT(st, excepthandler, s->v.Try.handlers, optimize); - VISIT_SEQ_OPT(st, stmt, s->v.Try.orelse, optimize); - VISIT_SEQ_OPT(st, stmt, s->v.Try.finalbody, optimize); + VISIT_SEQ(st, stmt, s->v.Try.body); + VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.orelse); + VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case TryStar_kind: - VISIT_SEQ_OPT(st, stmt, s->v.TryStar.body, optimize); - VISIT_SEQ_OPT(st, excepthandler, s->v.TryStar.handlers, optimize); - VISIT_SEQ_OPT(st, stmt, s->v.TryStar.orelse, optimize); - VISIT_SEQ_OPT(st, stmt, s->v.TryStar.finalbody, optimize); + VISIT_SEQ(st, stmt, s->v.TryStar.body); + VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); + VISIT_SEQ(st, stmt, s->v.TryStar.orelse); + VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); break; case Assert_kind: VISIT(st, expr, s->v.Assert.test); @@ -2162,7 +2151,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) break; case With_kind: VISIT_SEQ(st, withitem, s->v.With.items); - VISIT_SEQ_OPT(st, stmt, s->v.With.body, optimize); + VISIT_SEQ(st, stmt, s->v.With.body); break; case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) @@ -2193,7 +2182,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) return 0; } - if ((optimize < 2) && _PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { + if (_PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { new_ste->ste_has_docstring = 1; } @@ -2210,7 +2199,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); - VISIT_SEQ_OPT(st, stmt, s->v.AsyncFunctionDef.body, optimize); + VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); if (!symtable_exit_block(st)) return 0; if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { @@ -2225,7 +2214,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) return 0; } VISIT_SEQ(st, withitem, s->v.AsyncWith.items); - VISIT_SEQ_OPT(st, stmt, s->v.AsyncWith.body, optimize); + VISIT_SEQ(st, stmt, s->v.AsyncWith.body); break; case AsyncFor_kind: maybe_set_ste_coroutine_for_module(st, s); @@ -2234,9 +2223,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s, int optimize) } VISIT(st, expr, s->v.AsyncFor.target); VISIT(st, expr, s->v.AsyncFor.iter); - VISIT_SEQ_OPT(st, stmt, s->v.AsyncFor.body, optimize); + VISIT_SEQ(st, stmt, s->v.AsyncFor.body); if (s->v.AsyncFor.orelse) - VISIT_SEQ_OPT(st, stmt, s->v.AsyncFor.orelse, optimize); + VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse); break; } LEAVE_RECURSIVE(st); @@ -2819,14 +2808,14 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a) static int -symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh, int optimize) +symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) { if (eh->v.ExceptHandler.type) VISIT(st, expr, eh->v.ExceptHandler.type); if (eh->v.ExceptHandler.name) if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL, LOCATION(eh))) return 0; - VISIT_SEQ_OPT(st, stmt, eh->v.ExceptHandler.body, optimize); + VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body); return 1; } @@ -2841,13 +2830,13 @@ symtable_visit_withitem(struct symtable *st, withitem_ty item) } static int -symtable_visit_match_case(struct symtable *st, match_case_ty m, int optimize) +symtable_visit_match_case(struct symtable *st, match_case_ty m) { VISIT(st, pattern, m->pattern); if (m->guard) { VISIT(st, expr, m->guard); } - VISIT_SEQ_OPT(st, stmt, m->body, optimize); + VISIT_SEQ(st, stmt, m->body); return 1; } @@ -3090,7 +3079,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, return NULL; } future.ff_features |= flags->cf_flags; - st = _PySymtable_Build(mod, filename, &future, _Py_GetConfig()->optimization_level); + st = _PySymtable_Build(mod, filename, &future); _PyArena_Free(arena); return st; } From d08920a7210f0c1b1e7258a7b7d3ee1907f5d844 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 21:57:47 +0800 Subject: [PATCH 15/17] uniformly use `ste_has_docstring` to judge whether there is docstring --- Python/codegen.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 48942d0a71f6ab..d6c0b87ea79fc0 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -768,7 +768,8 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac /* If from __future__ import annotations is active, * every annotated class and module should have __annotations__. * Else __annotate__ is created when necessary. */ - if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && SYMTABLE_ENTRY(c)->ste_annotations_used) { + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && ste->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { @@ -776,8 +777,8 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac } Py_ssize_t first_instr = 0; if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */ - PyObject *docstring = _PyAST_GetDocString(stmts); - if (docstring) { + if (ste->ste_has_docstring) { + PyObject *docstring = _PyAST_GetDocString(stmts); first_instr = 1; /* set docstring */ assert(OPTIMIZATION_LEVEL(c) < 2); @@ -1241,9 +1242,8 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags PySTEntryObject *ste = SYMTABLE_ENTRY(c); Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(body); - assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); if (ste->ste_has_docstring) { + PyObject *docstring = _PyAST_GetDocString(body); assert(docstring); first_instr = 1; docstring = _PyCompile_CleanDoc(docstring); From 3c072f32d30035bab95feb49725298f295c07987 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 22:10:42 +0800 Subject: [PATCH 16/17] add `docstring` assert --- Python/codegen.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/codegen.c b/Python/codegen.c index d6c0b87ea79fc0..bce3b94b27a45d 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -779,6 +779,7 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */ if (ste->ste_has_docstring) { PyObject *docstring = _PyAST_GetDocString(stmts); + assert(docstring); first_instr = 1; /* set docstring */ assert(OPTIMIZATION_LEVEL(c) < 2); From 442d7a2b3f9e7bb4f7de9faf70ca902933259de5 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 8 Nov 2024 22:42:06 +0800 Subject: [PATCH 17/17] retrigger checks