diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 0a1c2f08cab3bd..ac7771041b38d4 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -22,6 +22,10 @@ Pending removal in Python 3.17 See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) +* :mod:`token`: + + - The :func:`token.ISTERMINAL`, :func:`token.ISNONTERMINAL` and + :func:`token.ISEOF` tokens. * :mod:`typing`: diff --git a/Doc/library/token.rst b/Doc/library/token.rst index c228006d4c1e1d..1e96b852310230 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -35,16 +35,22 @@ a ``"match"`` token may be either :data:`NAME` or :data:`SOFT_KEYWORD`. Return ``True`` for terminal token values. + .. deprecated-removed:: next 3.17 + .. function:: ISNONTERMINAL(x) Return ``True`` for non-terminal token values. + .. deprecated-removed:: next 3.17 + .. function:: ISEOF(x) Return ``True`` if *x* is the marker indicating the end of input. + .. deprecated-removed:: next 3.17 + The token constants are: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 295dc201ec0ae4..75e336346898a1 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -592,6 +592,15 @@ hashlib (Contributed by Bénédikt Tran in :gh:`134978`.) +token +----- + +* The :func:`token.ISTERMINAL`, :func:`token.ISNONTERMINAL` and + :func:`token.ISEOF` tokens are now deprecated, and will be removed + in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`86353`) + + .. Add deprecations above alphabetically, not here at the end. Removed diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index 5de1f719a2f1a2..eb3d96e8abe396 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -86,9 +86,9 @@ extern "C" { /* Special definitions for cooperation with parser */ -#define ISTERMINAL(x) ((x) < NT_OFFSET) -#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) -#define ISEOF(x) ((x) == ENDMARKER) +#define ISTERMINAL(x) ((x) < NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISEOF(x) ((x) == ENDMARKER) /* Deprecated and will be removed in 3.17 */ #define ISWHITESPACE(x) ((x) == ENDMARKER || \ (x) == NEWLINE || \ (x) == INDENT || \ diff --git a/Lib/token.py b/Lib/token.py index f61723cc09da02..2a5f6a6e533491 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -135,10 +135,19 @@ } def ISTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x < NT_OFFSET def ISNONTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISNONTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x >= NT_OFFSET def ISEOF(x: int) -> bool: + import warnings + warnings.warn('token.ISEOF is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x == ENDMARKER diff --git a/Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst b/Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst new file mode 100644 index 00000000000000..8d947689572b62 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst @@ -0,0 +1,2 @@ +Deprecate and schedule for removal in 3.17, :func:`token.ISTERMINAL`, +:func:`token.ISNONTERMINAL` and :func:`token.ISEOF`. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 9ee5ec86e75d47..88425f6114aa58 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -79,9 +79,9 @@ def update_file(file, content): /* Special definitions for cooperation with parser */ -#define ISTERMINAL(x) ((x) < NT_OFFSET) -#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) -#define ISEOF(x) ((x) == ENDMARKER) +#define ISTERMINAL(x) ((x) < NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISEOF(x) ((x) == ENDMARKER) /* Deprecated and will be removed in 3.17 */ #define ISWHITESPACE(x) ((x) == ENDMARKER || \\ (x) == NEWLINE || \\ (x) == INDENT || \\ @@ -279,12 +279,21 @@ def make_rst(infile, outfile='Doc/library/token-list.inc', } def ISTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x < NT_OFFSET def ISNONTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISNONTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x >= NT_OFFSET def ISEOF(x: int) -> bool: + import warnings + warnings.warn('token.ISEOF is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x == ENDMARKER '''