Skip to content
Open
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions clang/include/clang/Basic/IdentifierTable.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,55 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;

enum TokenKey : unsigned {
KEYC99 = 0x1,
KEYCXX = 0x2,
KEYCXX11 = 0x4,
KEYGNU = 0x8,
KEYMS = 0x10,
BOOLSUPPORT = 0x20,
KEYALTIVEC = 0x40,
KEYNOCXX = 0x80,
KEYBORLAND = 0x100,
KEYOPENCLC = 0x200,
KEYC23 = 0x400,
KEYNOMS18 = 0x800,
KEYNOOPENCL = 0x1000,
WCHARSUPPORT = 0x2000,
HALFSUPPORT = 0x4000,
CHAR8SUPPORT = 0x8000,
KEYOBJC = 0x10000,
KEYZVECTOR = 0x20000,
KEYCOROUTINES = 0x40000,
KEYMODULES = 0x80000,
KEYCXX20 = 0x100000,
KEYOPENCLCXX = 0x200000,
KEYMSCOMPAT = 0x400000,
KEYSYCL = 0x800000,
KEYCUDA = 0x1000000,
KEYZOS = 0x2000000,
KEYNOZOS = 0x4000000,
KEYHLSL = 0x8000000,
KEYFIXEDPOINT = 0x10000000,
KEYMAX = KEYFIXEDPOINT, // The maximum key
KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
};

/// How a keyword is treated in the selected standard. This enum is ordered
/// intentionally so that the value that 'wins' is the most 'permissive'.
enum KeywordStatus {
KS_Unknown, // Not yet calculated. Used when figuring out the status.
KS_Disabled, // Disabled
KS_Future, // Is a keyword in future standard
KS_Extension, // Is an extension
KS_Enabled, // Enabled
};

KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags);

enum class ReservedIdentifierStatus {
NotReserved = 0,
StartsWithUnderscoreAtGlobalScope,
Expand Down
53 changes: 1 addition & 52 deletions clang/lib/Basic/IdentifierTable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,57 +77,6 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
// Language Keyword Implementation
//===----------------------------------------------------------------------===//

// Constants for TokenKinds.def
namespace {

enum TokenKey : unsigned {
KEYC99 = 0x1,
KEYCXX = 0x2,
KEYCXX11 = 0x4,
KEYGNU = 0x8,
KEYMS = 0x10,
BOOLSUPPORT = 0x20,
KEYALTIVEC = 0x40,
KEYNOCXX = 0x80,
KEYBORLAND = 0x100,
KEYOPENCLC = 0x200,
KEYC23 = 0x400,
KEYNOMS18 = 0x800,
KEYNOOPENCL = 0x1000,
WCHARSUPPORT = 0x2000,
HALFSUPPORT = 0x4000,
CHAR8SUPPORT = 0x8000,
KEYOBJC = 0x10000,
KEYZVECTOR = 0x20000,
KEYCOROUTINES = 0x40000,
KEYMODULES = 0x80000,
KEYCXX20 = 0x100000,
KEYOPENCLCXX = 0x200000,
KEYMSCOMPAT = 0x400000,
KEYSYCL = 0x800000,
KEYCUDA = 0x1000000,
KEYZOS = 0x2000000,
KEYNOZOS = 0x4000000,
KEYHLSL = 0x8000000,
KEYFIXEDPOINT = 0x10000000,
KEYMAX = KEYFIXEDPOINT, // The maximum key
KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
};

/// How a keyword is treated in the selected standard. This enum is ordered
/// intentionally so that the value that 'wins' is the most 'permissive'.
enum KeywordStatus {
KS_Unknown, // Not yet calculated. Used when figuring out the status.
KS_Disabled, // Disabled
KS_Future, // Is a keyword in future standard
KS_Extension, // Is an extension
KS_Enabled, // Enabled
};

} // namespace

// This works on a single TokenKey flag and checks the LangOpts to get the
// KeywordStatus based exclusively on this flag, so that it can be merged in
// getKeywordStatus. Most should be enabled/disabled, but some might imply
Expand Down Expand Up @@ -222,7 +171,7 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts,

/// Translates flags as specified in TokenKinds.def into keyword status
/// in the given language standard.
static KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags) {
// KEYALL means always enabled, so special case this one.
if (Flags == KEYALL) return KS_Enabled;
Expand Down
20 changes: 13 additions & 7 deletions lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -719,9 +719,8 @@ CPlusPlusNameParser::ParseFullNameImpl() {
}
start_position.Remove();
return result;
} else {
return std::nullopt;
}
return std::nullopt;
}

llvm::StringRef CPlusPlusNameParser::GetTextForRange(const Range &range) {
Expand Down Expand Up @@ -755,12 +754,19 @@ static const clang::LangOptions &GetLangOptions() {
return g_options;
}

static const llvm::StringMap<tok::TokenKind> &GetKeywordsMap() {
static llvm::StringMap<tok::TokenKind> g_map{
#define KEYWORD(Name, Flags) {llvm::StringRef(#Name), tok::kw_##Name},
static const llvm::StringMap<tok::TokenKind> GetKeywordsMap() {
using namespace clang;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

very minor nit, but could we move this using namespace down to just before the macro. Just so it gives the reader some indication that we may be using it for the include machinery

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

llvm::StringMap<tok::TokenKind> g_map;

auto LangOpts = GetLangOptions();

KeywordStatus AddResult;
#define KEYWORD(NAME, FLAGS) \
AddResult = getKeywordStatus(LangOpts, FLAGS); \
if (AddResult != KS_Disabled) \
g_map[llvm::StringRef(#NAME)] = tok::kw_##NAME;
#include "clang/Basic/TokenKinds.def"
#undef KEYWORD
};
return g_map;
}

Expand All @@ -769,7 +775,7 @@ void CPlusPlusNameParser::ExtractTokens() {
return;
clang::Lexer lexer(clang::SourceLocation(), GetLangOptions(), m_text.data(),
m_text.data(), m_text.data() + m_text.size());
const auto &kw_map = GetKeywordsMap();
const auto kw_map = GetKeywordsMap();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a bit unfortunate we have to reconstruct this StringMap every time. Do you have an intuition for how expensive that is?

Why can't we construct the map statically? The LangOptions never change right? So you could just wrap all of this new logic in an immediately-invoked-lambda and still make the g_map static. Wdyt?

Copy link
Collaborator Author

@dsandersllvm dsandersllvm Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll see if I can measure it. I doubt it's noticeable for interactive use but it might add up in scripts

The LangOptions never change right?

At the moment they don't but I think that's a bug so I didn't add the call_once wrapper to it. I think they ought to depend on the active stack frame. For example:
target variable Class::requires
ought to work in a C++17 stack frame but not a C++20 stack frame. I'm not sure if it should work in an ObjC frame, I don't think it would at the moment but it's only supposed to be a keyword when it follows an @.

Along the same lines, if you have a C++20 frame call an OpenCL frame, this (admittedly contrived) example
target variable Class::_Atomic
should work in the OpenCL frame but not the C++20 frame. The reason I call this one contrived is because identifiers matching /^_[A-Z]/ are reserved by the standard.

clang::Token token;
for (lexer.LexFromRawLexer(token); !token.is(clang::tok::eof);
lexer.LexFromRawLexer(token)) {
Expand Down
Loading