From 80fb6d0cf5c78fa4bc759b444a9c1898e021ae80 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 6 Mar 2025 15:54:10 -0500 Subject: [PATCH 1/3] Support positional and keyword-only arguments Currently the signature parsing logic fails when confronted with a `/` or a `*`, rather than recognizing them as demarcating positional-only and keyword-only arguments. This patch supports parsing signatures with these features, but doesn't pass this information along to the `ArgSig` or `FunctionSig` classes, since the information would not be used anyway. --- mypy/stubdoc.py | 32 ++++++++++++++++++ mypy/test/teststubgen.py | 73 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 0da93b4e2477..1595ae056301 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -175,6 +175,8 @@ def __init__(self, function_name: str) -> None: self.ret_type = "Any" self.found = False self.args: list[ArgSig] = [] + self.pos_only: int | None = None + self.keyword_only: int | None = None # Valid signatures found so far. self.signatures: list[FunctionSig] = [] @@ -261,6 +263,15 @@ def add_token(self, token: tokenize.TokenInfo) -> None: return if token.string == ")": + if ( + self.state[-1] == STATE_ARGUMENT_LIST + and self.keyword_only is not None + and self.keyword_only == len(self.args) + and not self.arg_name + ): + # Error condition: * must be followed by arguments + self.reset() + return self.state.pop() # arg_name is empty when there are no args. e.g. func() @@ -280,6 +291,27 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.arg_type = None self.arg_default = None self.accumulator = "" + elif ( + token.type == tokenize.OP + and (token.string in {"*", "/"}) + and self.state[-1] == STATE_ARGUMENT_LIST + ): + if token.string == "/": + if self.pos_only is not None or self.keyword_only is not None or not self.args: + # Error cases: + # - / shows up more than once + # - / shows up after * + # - / shows up before any arguments + self.reset() + return + self.pos_only = len(self.args) + else: + if self.keyword_only is not None: + # * is not allowed after * + self.reset() + return + self.keyword_only = len(self.args) + self.state.append(STATE_ARGUMENT_TYPE) elif token.type == tokenize.OP and token.string == "->" and self.state[-1] == STATE_INIT: self.accumulator = "" diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 83693bebd91e..09f7ec13d2ed 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -399,6 +399,79 @@ def test_infer_sig_from_docstring_bad_indentation(self) -> None: None, ) + def test_infer_sig_from_docstring_positional_only_arguments(self) -> None: + assert_equal( + infer_sig_from_docstring("func(self, /) -> str", "func"), + [FunctionSig(name="func", args=[ArgSig(name="self")], ret_type="str")], + ) + + assert_equal( + infer_sig_from_docstring("func(self, x, /) -> str", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="self"), ArgSig(name="x")], ret_type="str" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("func(x, /, y) -> int", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="int")], + ) + + def test_infer_sig_from_docstring_keyword_only_arguments(self) -> None: + assert_equal( + infer_sig_from_docstring("func(*, x) -> str", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x")], ret_type="str")], + ) + + assert_equal( + infer_sig_from_docstring("func(x, *, y) -> str", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="str")], + ) + + assert_equal( + infer_sig_from_docstring("func(*, x, y) -> str", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="str")], + ) + + def test_infer_sig_from_docstring_pos_only_and_keyword_only_arguments(self) -> None: + assert_equal( + infer_sig_from_docstring("func(x, /, *, y) -> str", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="str")], + ) + + assert_equal( + infer_sig_from_docstring("func(x, /, y, *, z) -> str", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="x"), ArgSig(name="y"), ArgSig(name="z")], + ret_type="str", + ) + ], + ) + + def test_infer_sig_from_docstring_pos_only_and_keyword_only_arguments_errors(self) -> None: + # / as first argument + assert_equal(infer_sig_from_docstring("func(/, x) -> str", "func"), []) + + # * as last argument + assert_equal(infer_sig_from_docstring("func(x, *) -> str", "func"), []) + + # / after * + assert_equal(infer_sig_from_docstring("func(x, *, /, y) -> str", "func"), []) + + # Two / + assert_equal(infer_sig_from_docstring("func(x, /, /, *, y) -> str", "func"), []) + + assert_equal(infer_sig_from_docstring("func(x, /, y, /, *, z) -> str", "func"), []) + + # Two * + assert_equal(infer_sig_from_docstring("func(x, /, *, *, y) -> str", "func"), []) + + assert_equal(infer_sig_from_docstring("func(x, /, *, y, *, z) -> str", "func"), []) + def test_infer_arg_sig_from_anon_docstring(self) -> None: assert_equal( infer_arg_sig_from_anon_docstring("(*args, **kwargs)"), From 6eb593253cdb0f354524a0d805d04ae74d2ec118 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 6 Mar 2025 17:05:27 -0500 Subject: [PATCH 2/3] fixup! Support positional and keyword-only arguments --- mypy/stubdoc.py | 31 ++++++++++++++++++------------- mypy/test/teststubgen.py | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 1595ae056301..b03dfd94fa8b 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -254,13 +254,21 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.arg_type = self.accumulator self.state.pop() elif self.state[-1] == STATE_ARGUMENT_LIST: - self.arg_name = self.accumulator - if not ( - token.string == ")" and self.accumulator.strip() == "" - ) and not _ARG_NAME_RE.match(self.arg_name): - # Invalid argument name. - self.reset() - return + if self.accumulator == "*": + if self.keyword_only is not None: + # Error condition: cannot have * twice + self.reset() + return + self.keyword_only = len(self.args) + self.accumulator = "" + else: + self.arg_name = self.accumulator + if not ( + token.string == ")" and self.accumulator.strip() == "" + ) and not _ARG_NAME_RE.match(self.arg_name): + # Invalid argument name. + self.reset() + return if token.string == ")": if ( @@ -305,13 +313,10 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.reset() return self.pos_only = len(self.args) + self.state.append(STATE_ARGUMENT_TYPE) + self.accumulator = "" else: - if self.keyword_only is not None: - # * is not allowed after * - self.reset() - return - self.keyword_only = len(self.args) - self.state.append(STATE_ARGUMENT_TYPE) + self.accumulator = "*" elif token.type == tokenize.OP and token.string == "->" and self.state[-1] == STATE_INIT: self.accumulator = "" diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 09f7ec13d2ed..885e82fd9ba4 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -399,6 +399,28 @@ def test_infer_sig_from_docstring_bad_indentation(self) -> None: None, ) + def test_infer_sig_from_docstring_args_kwargs(self) -> None: + assert_equal( + infer_sig_from_docstring("func(*args, **kwargs) -> int", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], + ret_type="int", + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("func(*args) -> int", "func"), + [FunctionSig(name="func", args=[ArgSig(name="*args")], ret_type="int")], + ) + + assert_equal( + infer_sig_from_docstring("func(**kwargs) -> int", "func"), + [FunctionSig(name="func", args=[ArgSig(name="**kwargs")], ret_type="int")], + ) + def test_infer_sig_from_docstring_positional_only_arguments(self) -> None: assert_equal( infer_sig_from_docstring("func(self, /) -> str", "func"), From e4f564842fce46abe53775f11581582557db9b28 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 6 Mar 2025 22:17:16 -0500 Subject: [PATCH 3/3] fixup! Support positional and keyword-only arguments --- mypy/stubdoc.py | 6 ++-- mypy/test/teststubgen.py | 63 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index b03dfd94fa8b..617c5ecda408 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -262,6 +262,8 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.keyword_only = len(self.args) self.accumulator = "" else: + if self.accumulator.startswith("*"): + self.keyword_only = len(self.args) + 1 self.arg_name = self.accumulator if not ( token.string == ")" and self.accumulator.strip() == "" @@ -301,7 +303,7 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.accumulator = "" elif ( token.type == tokenize.OP - and (token.string in {"*", "/"}) + and token.string == "/" and self.state[-1] == STATE_ARGUMENT_LIST ): if token.string == "/": @@ -315,8 +317,6 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.pos_only = len(self.args) self.state.append(STATE_ARGUMENT_TYPE) self.accumulator = "" - else: - self.accumulator = "*" elif token.type == tokenize.OP and token.string == "->" and self.state[-1] == STATE_INIT: self.accumulator = "" diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 885e82fd9ba4..e45c4e42aba9 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -421,6 +421,19 @@ def test_infer_sig_from_docstring_args_kwargs(self) -> None: [FunctionSig(name="func", args=[ArgSig(name="**kwargs")], ret_type="int")], ) + @pytest.mark.xfail( + raises=AssertionError, reason="Arg and kwarg signature validation not implemented yet" + ) + def test_infer_sig_from_docstring_args_kwargs_errors(self) -> None: + # Double args + assert_equal(infer_sig_from_docstring("func(*args, *args2) -> int", "func"), []) + + # Double kwargs + assert_equal(infer_sig_from_docstring("func(**kw, **kw2) -> int", "func"), []) + + # args after kwargs + assert_equal(infer_sig_from_docstring("func(**kwargs, *args) -> int", "func"), []) + def test_infer_sig_from_docstring_positional_only_arguments(self) -> None: assert_equal( infer_sig_from_docstring("func(self, /) -> str", "func"), @@ -441,6 +454,26 @@ def test_infer_sig_from_docstring_positional_only_arguments(self) -> None: [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="int")], ) + assert_equal( + infer_sig_from_docstring("func(x, /, *args) -> str", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x"), ArgSig(name="*args")], ret_type="str" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("func(x, /, *, kwonly, **kwargs) -> str", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="x"), ArgSig(name="kwonly"), ArgSig(name="**kwargs")], + ret_type="str", + ) + ], + ) + def test_infer_sig_from_docstring_keyword_only_arguments(self) -> None: assert_equal( infer_sig_from_docstring("func(*, x) -> str", "func"), @@ -457,6 +490,17 @@ def test_infer_sig_from_docstring_keyword_only_arguments(self) -> None: [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="str")], ) + assert_equal( + infer_sig_from_docstring("func(x, *, kwonly, **kwargs) -> str", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="x"), ArgSig(name="kwonly"), ArgSig("**kwargs")], + ret_type="str", + ) + ], + ) + def test_infer_sig_from_docstring_pos_only_and_keyword_only_arguments(self) -> None: assert_equal( infer_sig_from_docstring("func(x, /, *, y) -> str", "func"), @@ -474,6 +518,22 @@ def test_infer_sig_from_docstring_pos_only_and_keyword_only_arguments(self) -> N ], ) + assert_equal( + infer_sig_from_docstring("func(x, /, y, *, z, **kwargs) -> str", "func"), + [ + FunctionSig( + name="func", + args=[ + ArgSig(name="x"), + ArgSig(name="y"), + ArgSig(name="z"), + ArgSig("**kwargs"), + ], + ret_type="str", + ) + ], + ) + def test_infer_sig_from_docstring_pos_only_and_keyword_only_arguments_errors(self) -> None: # / as first argument assert_equal(infer_sig_from_docstring("func(/, x) -> str", "func"), []) @@ -494,6 +554,9 @@ def test_infer_sig_from_docstring_pos_only_and_keyword_only_arguments_errors(sel assert_equal(infer_sig_from_docstring("func(x, /, *, y, *, z) -> str", "func"), []) + # *args and * are not allowed + assert_equal(infer_sig_from_docstring("func(*args, *, kwonly) -> str", "func"), []) + def test_infer_arg_sig_from_anon_docstring(self) -> None: assert_equal( infer_arg_sig_from_anon_docstring("(*args, **kwargs)"),