Regenerate the c++ keywords classifier

Author: robertoraggi

CMakeLists.txt

@@ -143,26 +143,6 @@ if (CXX_BUILD_TESTS)
 endif()
 
 
-if (NOT KWGEN_EXECUTABLE AND CMAKE_SYSTEM_NAME STREQUAL "WASI")
-
-    ExternalProject_Add(
-        kwgen_host
-        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tools/kwgen
-        INSTALL_COMMAND ""
-    )
-
-    ExternalProject_Get_property(kwgen_host BINARY_DIR)
-
-    add_executable(kwgen IMPORTED)
-    set_target_properties(kwgen PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/kwgen")
-    add_dependencies(kwgen kwgen_host)
-
-else()
-
-    add_subdirectory(tools/kwgen)
-
-endif()
-
 add_subdirectory(src)
 
 if (CXX_BUILD_TESTS)

Dockerfile.emsdk

@@ -5,10 +5,6 @@ RUN apt-get update && apt-get install -y \
     wget \
     unzip
 
-COPY tools/kwgen/kwgen.cc /tmp/kwgen.cc
-
-RUN g++ -std=c++20 /tmp/kwgen.cc -o /usr/bin/kwgen
-
 RUN wget https://github.com/google/flatbuffers/releases/download/v25.1.24/Linux.flatc.binary.clang++-18.zip && \
     unzip Linux.flatc.binary.clang++-18.zip && \
     mv flatc /usr/bin/flatc && \

Dockerfile.wasi

@@ -8,10 +8,6 @@ RUN apt-get update && apt-get install -y \
     unzip \
     ninja-build
 
-COPY tools/kwgen/kwgen.cc /tmp/kwgen.cc
-
-RUN g++ -std=c++20 /tmp/kwgen.cc -o /usr/bin/kwgen
-
 RUN wget https://github.com/google/flatbuffers/releases/download/v25.1.24/Linux.flatc.binary.clang++-18.zip && \
     unzip Linux.flatc.binary.clang++-18.zip && \
     mv flatc /usr/bin/flatc && \

packages/cxx-frontend/.prettierrc

@@ -0,0 +1 @@
+{}

packages/cxx-frontend/scripts/build.js

@@ -104,7 +104,6 @@ async function dockerBuild() {
 
   const cmakeOptions = [
     "-DCMAKE_INSTALL_PREFIX=build.em/install/usr",
-    "-DKWGEN_EXECUTABLE=/usr/bin/kwgen",
     "-DFLATBUFFERS_FLATC_EXECUTABLE=/usr/bin/flatc",
     "-S .",
     "-B build.em",
@@ -136,13 +135,12 @@ async function dockerBuild() {
   await $`${docker} run -t --rm -u ${user} -v ${emscriptenCacheDir}:/emsdk/upstream/emscripten/cache/ -v ${projectRootSourcePath}:/code -w /code cxx-emsdk cmake --build build.em`;
 }
 
-async function emsdkBuild({ cmake, emcmake, flatc, kwgen }) {
+async function emsdkBuild({ cmake, emcmake, flatc }) {
   const CMAKE_INTERPROCEDURAL_OPTIMIZATION =
     $.env.CMAKE_INTERPROCEDURAL_OPTIMIZATION ?? "ON";
 
   const cmakeOptions = [
     `-DCMAKE_INSTALL_PREFIX=${projectRootSourcePath}/build.em/install/usr`,
-    `-DKWGEN_EXECUTABLE=${kwgen}`,
     `-DFLATBUFFERS_FLATC_EXECUTABLE=${flatc}`,
     `-S ${projectRootSourcePath}`,
     `-B ${projectRootSourcePath}/build.em`,
@@ -193,16 +191,9 @@ async function detectEmsdk() {
     (await which("flatc", { nothrow: true }));
   if (!flatc) return null;
 
-  const kwgen =
-    argv.kwgen ??
-    $.env.KWGEN_EXECUTABLE ??
-    (await which("kwgen", { nothrow: true }));
-  if (!kwgen) return null;
-
   return {
     cmake,
     emcmake,
     flatc,
-    kwgen,
   };
 }

packages/cxx-gen-ast/.prettierrc

@@ -0,0 +1 @@
+{}

packages/cxx-gen-ast/src/gen.ts

@@ -44,6 +44,7 @@ import { gen_ast_slot_ts } from "./gen_ast_slot_ts.ts";
 import { gen_token_fwd_h } from "./gen_token_fwd_h.ts";
 import { gen_tokenkind_ts } from "./gen_tokenkind_ts.ts";
 import { gen_keywords_kwgen } from "./gen_keywords_kwgen.ts";
+import { gen_pp_keywords_kwgen } from "./gen_pp_keywords_kwgen.ts";
 import { gen_ast_pretty_printer_h } from "./gen_ast_pretty_printer_h.ts";
 import { gen_ast_pretty_printer_cc } from "./gen_ast_pretty_printer_cc.ts";
 
@@ -150,7 +151,10 @@ gen_tokenkind_ts({
   output: path.join(outdir, "packages/cxx-frontend/src/TokenKind.ts"),
 });
 gen_keywords_kwgen({
-  output: path.join(outdir, "src/parser/cxx/keywords.kwgen"),
+  output: path.join(outdir, "src/parser/cxx/private/keywords-priv.h"),
+});
+gen_pp_keywords_kwgen({
+  output: path.join(outdir, "src/parser/cxx/private/pp_directives-priv.h"),
 });
 
 child_process.execSync("clang-format -i *.h *.cc", {

packages/cxx-gen-ast/src/gen_keywords_kwgen.ts

@@ -18,32 +18,34 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-import * as fs from "fs";
 import * as tokens from "./tokens.ts";
+import kwgen from "./kwgen.ts";
+import { cpy_header } from "./cpy_header.ts";
 
 export function gen_keywords_kwgen({ output }: { output: string }) {
-  const code: string[] = [];
-  const emit = (line = "") => code.push(line);
-
   const isContextKeyword = (kw: string) => {
     return ["final", "override", "import", "module"].includes(kw);
   };
 
+  const keywords: string[] = [];
+
   tokens.KEYWORDS.filter((kw) => !isContextKeyword(kw)).forEach((tk) =>
-    emit(tk),
+    keywords.push(tk),
   );
 
-  emit();
-  tokens.TOKEN_ALIASES.forEach(([tk]) => emit(tk));
-
-  const out = `%no-enums
-%token-prefix=cxx::TokenKind::T_
-%token-type=cxx::TokenKind
-%toupper
-
-%%
-${code.join("\n")}
-`;
-
-  fs.writeFileSync(output, out);
+  tokens.TOKEN_ALIASES.forEach(([tk]) => {
+    keywords.push(tk);
+  });
+
+  kwgen({
+    copyright: cpy_header,
+    output,
+    keywords,
+    tokenPrefix: "cxx::TokenKind::T_",
+    tokenType: "cxx::TokenKind",
+    toUpper: true,
+    noEnums: true,
+    defaultToken: "cxx::TokenKind::T_IDENTIFIER",
+    classifier: "classify",
+  });
 }

packages/cxx-gen-ast/src/gen_pp_keywords_kwgen.ts

@@ -0,0 +1,53 @@
+// Copyright (c) 2025 Roberto Raggi <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+import kwgen from "./kwgen.ts";
+import { cpy_header } from "./cpy_header.ts";
+
+export function gen_pp_keywords_kwgen({ output }: { output: string }) {
+  kwgen({
+    copyright: cpy_header,
+    output,
+    keywords: [
+      "define",
+      "elif",
+      "elifdef",
+      "elifndef",
+      "else",
+      "endif",
+      "error",
+      "if",
+      "ifdef",
+      "ifndef",
+      "include_next",
+      "include",
+      "line",
+      "pragma",
+      "undef",
+      "warning",
+    ],
+    tokenPrefix: "PreprocessorDirectiveKind::T_",
+    tokenType: "PreprocessorDirectiveKind",
+    toUpper: true,
+    noEnums: false,
+    defaultToken: "PreprocessorDirectiveKind::T_IDENTIFIER",
+    classifier: "classifyDirective",
+  });
+}

packages/cxx-gen-ast/src/kwgen.ts

@@ -3,6 +3,7 @@ import { writeFileSync } from "node:fs";
 interface Options {
   keywords: string[];
   output: string | ((code: string) => void);
+  copyright?: string;
   noEnums: boolean;
   tokenPrefix: string;
   tokenType: string;
@@ -12,16 +13,50 @@ interface Options {
 }
 
 export default function kwgen(options: Options) {
-  const { keywords, output, tokenType, defaultToken, classifier } = options;
+  const {
+    copyright,
+    keywords,
+    output,
+    classifier,
+    defaultToken,
+    noEnums,
+    tokenPrefix,
+    tokenType,
+  } = options;
 
   let out: string[] = [];
   const emit = (s: string) => out.push(s);
 
+  if (copyright !== undefined) {
+    emit(copyright);
+  }
+
   const keywordsByLength = Map.groupBy(keywords, (s) => s.length);
 
   const lengths = Array.from(keywordsByLength.keys());
   lengths.sort((a, b) => a - b);
 
+  const getTokenName = (s: string) => {
+    const x = options.toUpper ? s.toUpperCase() : s;
+    let name = `${tokenPrefix}${x}`;
+    const scopeIndex = name.lastIndexOf("::");
+    if (scopeIndex !== -1) {
+      name = `${name.slice(scopeIndex + 2)}`;
+    }
+    return name;
+  };
+
+  if (!noEnums) {
+    emit(`enum class ${tokenType} {`);
+    emit(`  ${getTokenName(defaultToken)},`);
+    keywords.forEach((kw) => {
+      const name = getTokenName(kw);
+      emit(`  ${name},`);
+    });
+    emit(`};`);
+    emit("");
+  }
+
   for (const length of lengths) {
     const items = keywordsByLength.get(length)!;