From 48b6cb168f1f380b5d10a71542f9b288946a0181 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 24 Aug 2025 17:56:55 +0100 Subject: [PATCH 1/3] Commit --- Doc/deprecations/pending-removal-in-3.17.rst | 5 +++++ Doc/library/token.rst | 6 ++++++ Doc/whatsnew/3.15.rst | 9 +++++++++ Include/internal/pycore_token.h | 8 ++++---- Lib/token.py | 9 +++++++++ .../2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst | 2 ++ Tools/build/generate_token.py | 15 ++++++++++++--- 7 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 370b98307e5228..5110a560852bcd 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -8,3 +8,8 @@ Pending removal in Python 3.17 but it has been retained for backward compatibility, with removal scheduled for Python 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` and :func:`typing.get_args` instead of relying on private implementation details. + +* :mod:`token`: + + - The :func:`token.ISTERMINAL`, :func:`token.ISNONTERMINAL` and + :func:`token.ISEOF` tokens. diff --git a/Doc/library/token.rst b/Doc/library/token.rst index c228006d4c1e1d..1e96b852310230 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -35,16 +35,22 @@ a ``"match"`` token may be either :data:`NAME` or :data:`SOFT_KEYWORD`. Return ``True`` for terminal token values. + .. deprecated-removed:: next 3.17 + .. function:: ISNONTERMINAL(x) Return ``True`` for non-terminal token values. + .. deprecated-removed:: next 3.17 + .. function:: ISEOF(x) Return ``True`` if *x* is the marker indicating the end of input. + .. deprecated-removed:: next 3.17 + The token constants are: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 81aa12184ed35c..672fdadbd3ecac 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -440,6 +440,15 @@ hashlib (Contributed by Bénédikt Tran in :gh:`134978`.) +token +----- + +* The :func:`token.ISTERMINAL`, :func:`token.ISNONTERMINAL` and + :func:`token.ISEOF` tokens are now deprecated, and will be removed + in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`86353`) + + .. Add deprecations above alphabetically, not here at the end. Removed diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index 5de1f719a2f1a2..c02550cc2b1f0d 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -82,13 +82,13 @@ extern "C" { #define NL 66 #define ERRORTOKEN 67 #define N_TOKENS 69 -#define NT_OFFSET 256 +#define NT_OFFSET 256 /* Deprecated and will be removed in 3.17 */ /* Special definitions for cooperation with parser */ -#define ISTERMINAL(x) ((x) < NT_OFFSET) -#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) -#define ISEOF(x) ((x) == ENDMARKER) +#define ISTERMINAL(x) ((x) < NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISEOF(x) ((x) == ENDMARKER) /* Deprecated and will be removed in 3.17 */ #define ISWHITESPACE(x) ((x) == ENDMARKER || \ (x) == NEWLINE || \ (x) == INDENT || \ diff --git a/Lib/token.py b/Lib/token.py index f61723cc09da02..2a5f6a6e533491 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -135,10 +135,19 @@ } def ISTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x < NT_OFFSET def ISNONTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISNONTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x >= NT_OFFSET def ISEOF(x: int) -> bool: + import warnings + warnings.warn('token.ISEOF is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x == ENDMARKER diff --git a/Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst b/Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst new file mode 100644 index 00000000000000..8d947689572b62 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst @@ -0,0 +1,2 @@ +Deprecate and schedule for removal in 3.17, :func:`token.ISTERMINAL`, +:func:`token.ISNONTERMINAL` and :func:`token.ISEOF`. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 9ee5ec86e75d47..88425f6114aa58 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -79,9 +79,9 @@ def update_file(file, content): /* Special definitions for cooperation with parser */ -#define ISTERMINAL(x) ((x) < NT_OFFSET) -#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) -#define ISEOF(x) ((x) == ENDMARKER) +#define ISTERMINAL(x) ((x) < NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) /* Deprecated and will be removed in 3.17 */ +#define ISEOF(x) ((x) == ENDMARKER) /* Deprecated and will be removed in 3.17 */ #define ISWHITESPACE(x) ((x) == ENDMARKER || \\ (x) == NEWLINE || \\ (x) == INDENT || \\ @@ -279,12 +279,21 @@ def make_rst(infile, outfile='Doc/library/token-list.inc', } def ISTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x < NT_OFFSET def ISNONTERMINAL(x: int) -> bool: + import warnings + warnings.warn('token.ISNONTERMINAL is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x >= NT_OFFSET def ISEOF(x: int) -> bool: + import warnings + warnings.warn('token.ISEOF is deprecated and will be removed in 3.17', + DeprecationWarning, stacklevel=2) return x == ENDMARKER ''' From e760fdfef7ab714d176d0f5264b6c54cf7f69626 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 24 Aug 2025 17:57:48 +0100 Subject: [PATCH 2/3] Commit --- Include/internal/pycore_token.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index c02550cc2b1f0d..eb3d96e8abe396 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -82,7 +82,7 @@ extern "C" { #define NL 66 #define ERRORTOKEN 67 #define N_TOKENS 69 -#define NT_OFFSET 256 /* Deprecated and will be removed in 3.17 */ +#define NT_OFFSET 256 /* Special definitions for cooperation with parser */ From 611b974c1b36d31f31b8ef484f2e861e8c05d96d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 20 Sep 2025 12:34:25 +0100 Subject: [PATCH 3/3] !fixup merge --- Doc/deprecations/pending-removal-in-3.17.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 174ebdb8d07861..ac7771041b38d4 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -34,7 +34,6 @@ Pending removal in Python 3.17 but it has been retained for backward compatibility, with removal scheduled for Python 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` and :func:`typing.get_args` instead of relying on private implementation details. - - :class:`typing.ByteString`, deprecated since Python 3.9, is scheduled for removal in Python 3.17. @@ -53,4 +52,4 @@ Pending removal in Python 3.17 runtime or by static type checkers). See :pep:`PEP 688 <688#current-options>` for more details. - (Contributed by Shantanu Jain in :gh:`91896`.) \ No newline at end of file + (Contributed by Shantanu Jain in :gh:`91896`.)