Regenerate the c++ keywords classifier

Author: robertoraggi

CMakeLists.txt

@@ -143,26 +143,6 @@ if (CXX_BUILD_TESTS)
 endif()
 
 
-if (NOT KWGEN_EXECUTABLE AND CMAKE_SYSTEM_NAME STREQUAL "WASI")
-
-    ExternalProject_Add(
-        kwgen_host
-        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tools/kwgen
-        INSTALL_COMMAND ""
-    )
-
-    ExternalProject_Get_property(kwgen_host BINARY_DIR)
-
-    add_executable(kwgen IMPORTED)
-    set_target_properties(kwgen PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/kwgen")
-    add_dependencies(kwgen kwgen_host)
-
-else()
-
-    add_subdirectory(tools/kwgen)
-
-endif()
-
 add_subdirectory(src)
 
 if (CXX_BUILD_TESTS)

Dockerfile.emsdk

@@ -5,10 +5,6 @@ RUN apt-get update && apt-get install -y \
     wget \
     unzip
 
-COPY tools/kwgen/kwgen.cc /tmp/kwgen.cc
-
-RUN g++ -std=c++20 /tmp/kwgen.cc -o /usr/bin/kwgen
-
 RUN wget https://github.com/google/flatbuffers/releases/download/v25.1.24/Linux.flatc.binary.clang++-18.zip && \
     unzip Linux.flatc.binary.clang++-18.zip && \
     mv flatc /usr/bin/flatc && \

Dockerfile.wasi

@@ -8,10 +8,6 @@ RUN apt-get update && apt-get install -y \
     unzip \
     ninja-build
 
-COPY tools/kwgen/kwgen.cc /tmp/kwgen.cc
-
-RUN g++ -std=c++20 /tmp/kwgen.cc -o /usr/bin/kwgen
-
 RUN wget https://github.com/google/flatbuffers/releases/download/v25.1.24/Linux.flatc.binary.clang++-18.zip && \
     unzip Linux.flatc.binary.clang++-18.zip && \
     mv flatc /usr/bin/flatc && \

packages/cxx-frontend/.prettierrc

@@ -0,0 +1 @@
+{}

packages/cxx-frontend/scripts/build.js

@@ -104,7 +104,6 @@ async function dockerBuild() {
 
   const cmakeOptions = [
     "-DCMAKE_INSTALL_PREFIX=build.em/install/usr",
-    "-DKWGEN_EXECUTABLE=/usr/bin/kwgen",
     "-DFLATBUFFERS_FLATC_EXECUTABLE=/usr/bin/flatc",
     "-S .",
     "-B build.em",
@@ -136,13 +135,12 @@ async function dockerBuild() {
   await $`${docker} run -t --rm -u ${user} -v ${emscriptenCacheDir}:/emsdk/upstream/emscripten/cache/ -v ${projectRootSourcePath}:/code -w /code cxx-emsdk cmake --build build.em`;
 }
 
-async function emsdkBuild({ cmake, emcmake, flatc, kwgen }) {
+async function emsdkBuild({ cmake, emcmake, flatc }) {
   const CMAKE_INTERPROCEDURAL_OPTIMIZATION =
     $.env.CMAKE_INTERPROCEDURAL_OPTIMIZATION ?? "ON";
 
   const cmakeOptions = [
     `-DCMAKE_INSTALL_PREFIX=${projectRootSourcePath}/build.em/install/usr`,
-    `-DKWGEN_EXECUTABLE=${kwgen}`,
     `-DFLATBUFFERS_FLATC_EXECUTABLE=${flatc}`,
     `-S ${projectRootSourcePath}`,
     `-B ${projectRootSourcePath}/build.em`,
@@ -193,16 +191,9 @@ async function detectEmsdk() {
     (await which("flatc", { nothrow: true }));
   if (!flatc) return null;
 
-  const kwgen =
-    argv.kwgen ??
-    $.env.KWGEN_EXECUTABLE ??
-    (await which("kwgen", { nothrow: true }));
-  if (!kwgen) return null;
-
   return {
     cmake,
     emcmake,
     flatc,
-    kwgen,
   };
 }

packages/cxx-frontend/src/TokenKind.ts

@@ -88,6 +88,31 @@ export enum TokenKind {
   STAR_EQUAL,
   STAR,
   TILDE,
+  _ATOMIC,
+  _BITINT,
+  _COMPLEX,
+  _DECIMAL128,
+  _DECIMAL32,
+  _DECIMAL64,
+  _GENERIC,
+  _IMAGINARY,
+  _NORETURN,
+  __ATTRIBUTE__,
+  __BUILTIN_BIT_CAST,
+  __BUILTIN_OFFSETOF,
+  __BUILTIN_VA_ARG,
+  __BUILTIN_VA_LIST,
+  __COMPLEX__,
+  __EXTENSION__,
+  __FLOAT128,
+  __FLOAT80,
+  __IMAG__,
+  __INT128,
+  __INT64,
+  __REAL__,
+  __RESTRICT__,
+  __THREAD,
+  __UNDERLYING_TYPE,
   ALIGNAS,
   ALIGNOF,
   ASM,
@@ -105,8 +130,8 @@ export enum TokenKind {
   CO_RETURN,
   CO_YIELD,
   CONCEPT,
-  CONST_CAST,
   CONST,
+  CONST_CAST,
   CONSTEVAL,
   CONSTEXPR,
   CONSTINIT,
@@ -142,15 +167,17 @@ export enum TokenKind {
   PRIVATE,
   PROTECTED,
   PUBLIC,
+  REGISTER,
   REINTERPRET_CAST,
   REQUIRES,
+  RESTRICT,
   RETURN,
   SHORT,
   SIGNED,
   SIZEOF,
+  STATIC,
   STATIC_ASSERT,
   STATIC_CAST,
-  STATIC,
   STRUCT,
   SWITCH,
   TEMPLATE,
@@ -162,6 +189,8 @@ export enum TokenKind {
   TYPEDEF,
   TYPEID,
   TYPENAME,
+  TYPEOF,
+  TYPEOF_UNQUAL,
   UNION,
   UNSIGNED,
   USING,
@@ -170,22 +199,4 @@ export enum TokenKind {
   VOLATILE,
   WCHAR_T,
   WHILE,
-  __ATTRIBUTE__,
-  __BUILTIN_BIT_CAST,
-  __BUILTIN_OFFSETOF,
-  __BUILTIN_VA_ARG,
-  __BUILTIN_VA_LIST,
-  __COMPLEX__,
-  __EXTENSION__,
-  __FLOAT128,
-  __FLOAT80,
-  __IMAG__,
-  __INT128,
-  __INT64,
-  __REAL__,
-  __RESTRICT__,
-  __THREAD,
-  __UNDERLYING_TYPE,
-  _ATOMIC,
-  _COMPLEX,
 }

packages/cxx-gen-ast/.prettierrc

@@ -0,0 +1 @@
+{}

packages/cxx-gen-ast/src/gen.ts

@@ -44,6 +44,8 @@ import { gen_ast_slot_ts } from "./gen_ast_slot_ts.ts";
 import { gen_token_fwd_h } from "./gen_token_fwd_h.ts";
 import { gen_tokenkind_ts } from "./gen_tokenkind_ts.ts";
 import { gen_keywords_kwgen } from "./gen_keywords_kwgen.ts";
+import { gen_c_keywords_kwgen } from "./gen_c_keywords_kwgen.ts";
+import { gen_pp_keywords_kwgen } from "./gen_pp_keywords_kwgen.ts";
 import { gen_ast_pretty_printer_h } from "./gen_ast_pretty_printer_h.ts";
 import { gen_ast_pretty_printer_cc } from "./gen_ast_pretty_printer_cc.ts";
 
@@ -150,7 +152,13 @@ gen_tokenkind_ts({
   output: path.join(outdir, "packages/cxx-frontend/src/TokenKind.ts"),
 });
 gen_keywords_kwgen({
-  output: path.join(outdir, "src/parser/cxx/keywords.kwgen"),
+  output: path.join(outdir, "src/parser/cxx/private/keywords-priv.h"),
+});
+gen_c_keywords_kwgen({
+  output: path.join(outdir, "src/parser/cxx/private/c_keywords-priv.h"),
+});
+gen_pp_keywords_kwgen({
+  output: path.join(outdir, "src/parser/cxx/private/pp_directives-priv.h"),
 });
 
 child_process.execSync("clang-format -i *.h *.cc", {

packages/cxx-gen-ast/src/gen_c_keywords_kwgen.ts

@@ -0,0 +1,45 @@
+// Copyright (c) 2025 Roberto Raggi <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+import * as tokens from "./tokens.ts";
+import kwgen from "./kwgen.ts";
+import { cpy_header } from "./cpy_header.ts";
+
+export function gen_c_keywords_kwgen({ output }: { output: string }) {
+  const keywords: string[] = [];
+
+  keywords.push(...tokens.C_KEYWORDS);
+
+  Object.entries(tokens.C_TOKEN_ALIASES).forEach(([tk]) => {
+    keywords.push(tk);
+  });
+
+  kwgen({
+    copyright: cpy_header,
+    output,
+    keywords,
+    tokenPrefix: "cxx::TokenKind::T_",
+    tokenType: "cxx::TokenKind",
+    toUpper: true,
+    noEnums: true,
+    defaultToken: "cxx::TokenKind::T_IDENTIFIER",
+    classifier: "classifyC",
+  });
+}

packages/cxx-gen-ast/src/gen_keywords_kwgen.ts

@@ -18,32 +18,34 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-import * as fs from "fs";
 import * as tokens from "./tokens.ts";
+import kwgen from "./kwgen.ts";
+import { cpy_header } from "./cpy_header.ts";
 
 export function gen_keywords_kwgen({ output }: { output: string }) {
-  const code: string[] = [];
-  const emit = (line = "") => code.push(line);
-
   const isContextKeyword = (kw: string) => {
     return ["final", "override", "import", "module"].includes(kw);
   };
 
-  tokens.KEYWORDS.filter((kw) => !isContextKeyword(kw)).forEach((tk) =>
-    emit(tk),
-  );
-
-  emit();
-  tokens.TOKEN_ALIASES.forEach(([tk]) => emit(tk));
+  const keywords: string[] = [];
 
-  const out = `%no-enums
-%token-prefix=cxx::TokenKind::T_
-%token-type=cxx::TokenKind
-%toupper
-
-%%
-${code.join("\n")}
-`;
+  tokens.CXX_KEYWORDS.filter((kw) => !isContextKeyword(kw)).forEach((tk) =>
+    keywords.push(tk),
+  );
 
-  fs.writeFileSync(output, out);
+  Object.entries(tokens.CXX_TOKEN_ALIASES).forEach(([tk]) => {
+    keywords.push(tk);
+  });
+
+  kwgen({
+    copyright: cpy_header,
+    output,
+    keywords,
+    tokenPrefix: "cxx::TokenKind::T_",
+    tokenType: "cxx::TokenKind",
+    toUpper: true,
+    noEnums: true,
+    defaultToken: "cxx::TokenKind::T_IDENTIFIER",
+    classifier: "classify",
+  });
 }