Add basic support to recognize C keywords

Author: robertoraggi

packages/cxx-gen-ast/src/tokens.ts

@@ -278,6 +278,7 @@ export const CXX_TOKEN_ALIASES = {
 };
 
 export const C_KEYWORDS: string[] = [
+  "asm",
   "alignas",
   "alignof",
   "auto",
@@ -332,11 +333,35 @@ export const C_KEYWORDS: string[] = [
   "_Generic",
   "_Imaginary",
   "_Noreturn",
+
+  "__attribute__",
+  "__builtin_bit_cast",
+  "__builtin_offsetof",
+  "__builtin_va_arg",
+  "__builtin_va_list",
+  "__complex__",
+  "__extension__",
+  "__float128",
+  "__float80",
+  "__imag__",
+  "__int128",
+  "__int64",
+  "__real__",
+  "__restrict__",
+  "__thread",
+  "__underlying_type",
+  "_Atomic",
+  "_Complex",
 ];
 
 export const C_TOKEN_ALIASES = {
+  __asm__: "ASM",
+  __asm: "ASM",
+  __inline__: "INLINE",
+  __inline: "INLINE",
   _Alignas: "ALIGNAS",
   _Alignof: "ALIGNOF",
+  _asm: "ASM",
   _Bool: "BOOL",
   _Static_assert: "STATIC_ASSERT",
   _Thread_local: "THREAD_LOCAL",

src/frontend/cxx/frontend.cc

@@ -117,6 +117,9 @@ auto readAll(const std::string& fileName) -> std::optional<std::string> {
 }
 
 void dumpTokens(const CLI& cli, TranslationUnit& unit, std::ostream& output) {
+  const auto lang =
+      cli.getSingle("-x") == "c" ? LanguageKind::kC : LanguageKind::kCXX;
+
   std::string flags;
 
   for (SourceLocation loc(1);; loc = loc.next()) {
@@ -134,7 +137,7 @@ void dumpTokens(const CLI& cli, TranslationUnit& unit, std::ostream& output) {
 
     auto kind = tk.kind();
     if (kind == TokenKind::T_IDENTIFIER) {
-      kind = Lexer::classifyKeyword(tk.spell());
+      kind = Lexer::classifyKeyword(tk.spell(), lang);
     }
 
     output << std::format("{} '{}'{}", Token::name(kind), tk.spell(), flags);
@@ -153,6 +156,13 @@ auto runOnFile(const CLI& cli, const std::string& fileName) -> bool {
 
   auto preprocessor = unit.preprocessor();
 
+  const auto lang = cli.getSingle("-x");
+
+  if (lang == "c") {
+    // set the language to C
+    preprocessor->setLanguage(LanguageKind::kC);
+  }
+
   std::unique_ptr<Toolchain> toolchain;
 
   if (cli.opt_verify) {

src/parser/cxx/cli.cc

@@ -139,7 +139,7 @@ std::vector<CLIOptionDescr> options{
     {"-o", "<file>", "Place output into <file>",
      CLIOptionDescrKind::kSeparated},
 
-    {"-x", "Specify the language from the compiler driver",
+    {"-x", "Specify the language from the compiler driver, e.g. c, or c++",
      CLIOptionDescrKind::kSeparated},
 
     {"-fcheck", "Enable type checker (WIP)", &CLI::opt_fcheck},

src/parser/cxx/cxx_fwd.h

@@ -28,6 +28,11 @@ class Arena;
 class Control;
 class TranslationUnit;
 
+enum struct LanguageKind {
+  kC,
+  kCXX,
+};
+
 [[noreturn]] void cxx_runtime_error(const std::string& msg);
 
 }  // namespace cxx

src/parser/cxx/lexer.cc

@@ -107,17 +107,18 @@ inline auto skipBOM(It& it, It end) -> bool {
 }
 }  // namespace
 
-Lexer::Lexer(std::string_view source)
-    : source_(source), pos_(cbegin(source_)), end_(cend(source_)) {
+Lexer::Lexer(std::string_view source, LanguageKind lang)
+    : source_(source), pos_(cbegin(source_)), end_(cend(source_)), lang_(lang) {
   hasBOM_ = skipBOM(pos_, end_);
   currentChar_ = pos_ < end_ ? peekNext(pos_, end_) : 0;
 }
 
-Lexer::Lexer(std::string buffer)
+Lexer::Lexer(std::string buffer, LanguageKind lang)
     : buffer_(std::move(buffer)),
       source_(buffer_),
       pos_(cbegin(source_)),
-      end_(cend(source_)) {
+      end_(cend(source_)),
+      lang_(lang) {
   hasBOM_ = skipBOM(pos_, end_);
   currentChar_ = pos_ < end_ ? peekNext(pos_, end_) : 0;
 }
@@ -206,13 +207,7 @@ auto Lexer::readToken() -> TokenKind {
 
       if (preprocessing_) return TokenKind::T_IDENTIFIER;
 
-      if (auto keyword =
-              classify(text_.c_str(), static_cast<int>(text_.length()));
-          keyword != TokenKind::T_IDENTIFIER) {
-        return keyword;
-      }
-
-      return TokenKind::T_IDENTIFIER;
+      return classifyKeyword(text_, lang_);
     }
   }
 
@@ -258,15 +253,17 @@ auto Lexer::readToken() -> TokenKind {
       }
     }
 
-    bool ud = false;
-    if (std::isalpha(LA()) || LA() == '_') {
-      ud = true;
-      do {
-        consume();
-      } while (pos_ != end_ && is_idcont(LA()));
-    }
+    if (lang_ == LanguageKind::kCXX) {
+      bool ud = false;
+      if (std::isalpha(LA()) || LA() == '_') {
+        ud = true;
+        do {
+          consume();
+        } while (pos_ != end_ && is_idcont(LA()));
+      }
 
-    if (ud) return TokenKind::T_USER_DEFINED_STRING_LITERAL;
+      if (ud) return TokenKind::T_USER_DEFINED_STRING_LITERAL;
+    }
 
     switch (encodingPrefix) {
       case EncodingPrefix::kWide:
@@ -569,8 +566,12 @@ auto Lexer::skipSpaces() -> bool {
   return pos_ != end_;
 }
 
-auto Lexer::classifyKeyword(const std::string_view& text) -> TokenKind {
-  return classify(text.data(), static_cast<int>(text.size()));
+auto Lexer::classifyKeyword(const std::string_view& text, LanguageKind lang)
+    -> TokenKind {
+  if (lang == LanguageKind::kCXX) {
+    return classify(text.data(), static_cast<int>(text.size()));
+  }
+  return classifyC(text.data(), static_cast<int>(text.size()));
 }
 
 void Lexer::clearBuffer() { buffer_.clear(); }

src/parser/cxx/lexer.h

@@ -29,8 +29,10 @@ namespace cxx {
 
 class Lexer {
  public:
-  explicit Lexer(std::string_view source);
-  explicit Lexer(std::string buffer);
+  explicit Lexer(std::string_view source,
+                 LanguageKind lang = LanguageKind::kCXX);
+
+  explicit Lexer(std::string buffer, LanguageKind lang = LanguageKind::kCXX);
 
   [[nodiscard]] auto preprocessing() const -> bool { return preprocessing_; }
   void setPreprocessing(bool preprocessing) { preprocessing_ = preprocessing; }
@@ -77,7 +79,8 @@ class Lexer {
   [[nodiscard]] auto text() -> std::string& { return text_; }
   [[nodiscard]] auto text() const -> const std::string& { return text_; }
 
-  static auto classifyKeyword(const std::string_view& text) -> TokenKind;
+  static auto classifyKeyword(const std::string_view& text, LanguageKind lang)
+      -> TokenKind;
 
   struct State {
     std::string_view::const_iterator pos_;
@@ -114,6 +117,7 @@ class Lexer {
   std::string_view::const_iterator pos_;
   std::string_view::const_iterator end_;
   std::string text_;
+  LanguageKind lang_ = LanguageKind::kCXX;
   bool leadingSpace_ = false;
   bool startOfLine_ = true;
   bool keepComments_ = false;

src/parser/cxx/preprocessor.cc

@@ -701,6 +701,7 @@ struct Preprocessor::Private {
   Control *control_ = nullptr;
   DiagnosticsClient *diagnosticsClient_ = nullptr;
   CommentHandler *commentHandler_ = nullptr;
+  LanguageKind language_ = LanguageKind::kCXX;
   bool canResolveFiles_ = true;
   std::vector<std::string> systemIncludePaths_;
   std::vector<std::string> quoteIncludePaths_;
@@ -1621,7 +1622,7 @@ void Preprocessor::Private::finalizeToken(std::vector<Token> &tokens,
 
   switch (tk->kind) {
     case TokenKind::T_IDENTIFIER: {
-      kind = Lexer::classifyKeyword(tk->text);
+      kind = Lexer::classifyKeyword(tk->text, language_);
 
       if (kind == TokenKind::T_IDENTIFIER) {
         value.idValue = control_->getIdentifier(tk->text);
@@ -1710,7 +1711,7 @@ void Preprocessor::Private::finalizeToken(std::vector<Token> &tokens,
 
 auto Preprocessor::Private::tokenize(const std::string_view &source,
                                      int sourceFile, bool bol) -> TokList * {
-  cxx::Lexer lex(source);
+  cxx::Lexer lex(source, language_);
   lex.setKeepComments(true);
   lex.setPreprocessing(true);
   TokList *ts = nullptr;
@@ -2824,7 +2825,7 @@ auto Preprocessor::Private::merge(const Tok *left, const Tok *right)
   if (!right) return left;
   const auto hideset = makeIntersection(left->hideset, right->hideset);
   auto text = string(std::string(left->text) + std::string(right->text));
-  Lexer lex(text);
+  Lexer lex(text, language_);
   lex.setPreprocessing(true);
   lex.next();
   auto tok = gen(lex.tokenKind(), lex.tokenText(), hideset);
@@ -2925,6 +2926,10 @@ auto Preprocessor::diagnosticsClient() const -> DiagnosticsClient * {
   return d->diagnosticsClient_;
 }
 
+auto Preprocessor::language() const -> LanguageKind { return d->language_; }
+
+void Preprocessor::setLanguage(LanguageKind lang) { d->language_ = lang; }
+
 auto Preprocessor::commentHandler() const -> CommentHandler * {
   return d->commentHandler_;
 }
@@ -3091,7 +3096,7 @@ void Preprocessor::getPreprocessedText(const std::vector<Token> &tokens,
       const auto &prevToken = tokens[index - 2];
       std::string s = prevToken.spell();
       s += token.spell();
-      Lexer lex(s);
+      Lexer lex(s, d->language_);
       // lex.setPreprocessing(true);
       lex.next();
       if (lex.tokenKind() != prevToken.kind()) {

src/parser/cxx/preprocessor.h

@@ -58,6 +58,11 @@ class Preprocessor {
 
   [[nodiscard]] auto diagnosticsClient() const -> DiagnosticsClient *;
 
+  [[nodiscard]] auto language() const -> LanguageKind;
+  void setLanguage(LanguageKind lang);
+
+  [[nodiscard]] auto preprocessorDelegate() const -> PreprocessorDelegate *;
+
   [[nodiscard]] auto commentHandler() const -> CommentHandler *;
   void setCommentHandler(CommentHandler *commentHandler);