From 8e0c1a7a39447f7785e6301ad54307b3f3479a48 Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Fri, 17 Oct 2025 18:10:51 -0700 Subject: [PATCH 1/4] [clang] Refactor clang's keyword enable/disable mechanism to allow lldb to re-use it lldb's CPlusPlusNameParser is currently identifying keywords using it's own map implemented using clang/Basic/TokenKinds.def. However, it does not respect the language options so identifiers can incorrectly determined to be keywords when using languages in which they are not keywords. Rather than implement the logic to enable/disable keywords in both projects it makes sense for lldb to use clang's implementation. See #164284 for more information --- clang/include/clang/Basic/IdentifierTable.h | 49 +++++++++++++++++++ clang/lib/Basic/IdentifierTable.cpp | 53 +-------------------- 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h index e4044bcdfcc60..54500b95b2c06 100644 --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -46,6 +46,55 @@ class LangOptions; class MultiKeywordSelector; class SourceLocation; +enum TokenKey : unsigned { + KEYC99 = 0x1, + KEYCXX = 0x2, + KEYCXX11 = 0x4, + KEYGNU = 0x8, + KEYMS = 0x10, + BOOLSUPPORT = 0x20, + KEYALTIVEC = 0x40, + KEYNOCXX = 0x80, + KEYBORLAND = 0x100, + KEYOPENCLC = 0x200, + KEYC23 = 0x400, + KEYNOMS18 = 0x800, + KEYNOOPENCL = 0x1000, + WCHARSUPPORT = 0x2000, + HALFSUPPORT = 0x4000, + CHAR8SUPPORT = 0x8000, + KEYOBJC = 0x10000, + KEYZVECTOR = 0x20000, + KEYCOROUTINES = 0x40000, + KEYMODULES = 0x80000, + KEYCXX20 = 0x100000, + KEYOPENCLCXX = 0x200000, + KEYMSCOMPAT = 0x400000, + KEYSYCL = 0x800000, + KEYCUDA = 0x1000000, + KEYZOS = 0x2000000, + KEYNOZOS = 0x4000000, + KEYHLSL = 0x8000000, + KEYFIXEDPOINT = 0x10000000, + KEYMAX = KEYFIXEDPOINT, // The maximum key + KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20, + KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL & + ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded. +}; + +/// How a keyword is treated in the selected standard. This enum is ordered +/// intentionally so that the value that 'wins' is the most 'permissive'. +enum KeywordStatus { + KS_Unknown, // Not yet calculated. Used when figuring out the status. + KS_Disabled, // Disabled + KS_Future, // Is a keyword in future standard + KS_Extension, // Is an extension + KS_Enabled, // Enabled +}; + +KeywordStatus getKeywordStatus(const LangOptions &LangOpts, + unsigned Flags); + enum class ReservedIdentifierStatus { NotReserved = 0, StartsWithUnderscoreAtGlobalScope, diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp index 4a2b77cd16bfc..65ed1273350c5 100644 --- a/clang/lib/Basic/IdentifierTable.cpp +++ b/clang/lib/Basic/IdentifierTable.cpp @@ -77,57 +77,6 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts, // Language Keyword Implementation //===----------------------------------------------------------------------===// -// Constants for TokenKinds.def -namespace { - -enum TokenKey : unsigned { - KEYC99 = 0x1, - KEYCXX = 0x2, - KEYCXX11 = 0x4, - KEYGNU = 0x8, - KEYMS = 0x10, - BOOLSUPPORT = 0x20, - KEYALTIVEC = 0x40, - KEYNOCXX = 0x80, - KEYBORLAND = 0x100, - KEYOPENCLC = 0x200, - KEYC23 = 0x400, - KEYNOMS18 = 0x800, - KEYNOOPENCL = 0x1000, - WCHARSUPPORT = 0x2000, - HALFSUPPORT = 0x4000, - CHAR8SUPPORT = 0x8000, - KEYOBJC = 0x10000, - KEYZVECTOR = 0x20000, - KEYCOROUTINES = 0x40000, - KEYMODULES = 0x80000, - KEYCXX20 = 0x100000, - KEYOPENCLCXX = 0x200000, - KEYMSCOMPAT = 0x400000, - KEYSYCL = 0x800000, - KEYCUDA = 0x1000000, - KEYZOS = 0x2000000, - KEYNOZOS = 0x4000000, - KEYHLSL = 0x8000000, - KEYFIXEDPOINT = 0x10000000, - KEYMAX = KEYFIXEDPOINT, // The maximum key - KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20, - KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL & - ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded. -}; - -/// How a keyword is treated in the selected standard. This enum is ordered -/// intentionally so that the value that 'wins' is the most 'permissive'. -enum KeywordStatus { - KS_Unknown, // Not yet calculated. Used when figuring out the status. - KS_Disabled, // Disabled - KS_Future, // Is a keyword in future standard - KS_Extension, // Is an extension - KS_Enabled, // Enabled -}; - -} // namespace - // This works on a single TokenKey flag and checks the LangOpts to get the // KeywordStatus based exclusively on this flag, so that it can be merged in // getKeywordStatus. Most should be enabled/disabled, but some might imply @@ -222,7 +171,7 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts, /// Translates flags as specified in TokenKinds.def into keyword status /// in the given language standard. -static KeywordStatus getKeywordStatus(const LangOptions &LangOpts, +KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts, unsigned Flags) { // KEYALL means always enabled, so special case this one. if (Flags == KEYALL) return KS_Enabled; From 2c7391fadb211cbea19bf423bbf8064638cbfa78 Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Mon, 27 Oct 2025 14:59:59 -0700 Subject: [PATCH 2/4] fixup: formatting --- clang/include/clang/Basic/IdentifierTable.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h index 54500b95b2c06..7062aea3100f3 100644 --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -92,8 +92,7 @@ enum KeywordStatus { KS_Enabled, // Enabled }; -KeywordStatus getKeywordStatus(const LangOptions &LangOpts, - unsigned Flags); +KeywordStatus getKeywordStatus(const LangOptions &LangOpts, unsigned Flags); enum class ReservedIdentifierStatus { NotReserved = 0, From af4a46ff22c1e4f565aadca98b29e409d7cf8574 Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Tue, 4 Nov 2025 15:45:43 -0800 Subject: [PATCH 3/4] fixup: Move getKeywordStatus's comment to the header --- clang/include/clang/Basic/IdentifierTable.h | 2 ++ clang/lib/Basic/IdentifierTable.cpp | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h index 7062aea3100f3..24d51b36df640 100644 --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -92,6 +92,8 @@ enum KeywordStatus { KS_Enabled, // Enabled }; +/// Translates flags as specified in TokenKinds.def into keyword status +/// in the given language standard. KeywordStatus getKeywordStatus(const LangOptions &LangOpts, unsigned Flags); enum class ReservedIdentifierStatus { diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp index 65ed1273350c5..d1c959b9687c4 100644 --- a/clang/lib/Basic/IdentifierTable.cpp +++ b/clang/lib/Basic/IdentifierTable.cpp @@ -169,8 +169,6 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts, } } -/// Translates flags as specified in TokenKinds.def into keyword status -/// in the given language standard. KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts, unsigned Flags) { // KEYALL means always enabled, so special case this one. From 6a80842c253cc8b425cc5fa03291993aa1708a25 Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Tue, 4 Nov 2025 16:04:02 -0800 Subject: [PATCH 4/4] fixup: Restore comment for TokenKey This comment was originally on the namespace containing TokenKey and KeywordStatus. I've only restored it to TokenKey because KeywordStatus is used by the code that uses TokenKinds.def not TokenKinds.def itself. --- clang/include/clang/Basic/IdentifierTable.h | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h index 24d51b36df640..b27492d19a65b 100644 --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -46,6 +46,7 @@ class LangOptions; class MultiKeywordSelector; class SourceLocation; +/// Constants for TokenKinds.def enum TokenKey : unsigned { KEYC99 = 0x1, KEYCXX = 0x2,