Regenerate the c++ keywords classifier

Author: robertoraggi

packages/cxx-gen-ast/src/gen.ts

@@ -150,7 +150,7 @@ gen_tokenkind_ts({
   output: path.join(outdir, "packages/cxx-frontend/src/TokenKind.ts"),
 });
 gen_keywords_kwgen({
-  output: path.join(outdir, "src/parser/cxx/keywords.kwgen"),
+  output: path.join(outdir, "src/parser/cxx/private/keywords-priv.h"),
 });
 
 child_process.execSync("clang-format -i *.h *.cc", {

packages/cxx-gen-ast/src/gen_keywords_kwgen.ts

@@ -18,32 +18,32 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-import * as fs from "fs";
 import * as tokens from "./tokens.ts";
+import kwgen from "./kwgen.ts";
 
 export function gen_keywords_kwgen({ output }: { output: string }) {
-  const code: string[] = [];
-  const emit = (line = "") => code.push(line);
-
   const isContextKeyword = (kw: string) => {
     return ["final", "override", "import", "module"].includes(kw);
   };
 
+  const keywords: string[] = [];
+
   tokens.KEYWORDS.filter((kw) => !isContextKeyword(kw)).forEach((tk) =>
-    emit(tk),
+    keywords.push(tk),
   );
 
-  emit();
-  tokens.TOKEN_ALIASES.forEach(([tk]) => emit(tk));
-
-  const out = `%no-enums
-%token-prefix=cxx::TokenKind::T_
-%token-type=cxx::TokenKind
-%toupper
-
-%%
-${code.join("\n")}
-`;
-
-  fs.writeFileSync(output, out);
+  tokens.TOKEN_ALIASES.forEach(([tk]) => {
+    keywords.push(tk);
+  });
+
+  kwgen({
+    output,
+    keywords,
+    tokenPrefix: "cxx::TokenKind::T_",
+    tokenType: "cxx::TokenKind",
+    toUpper: true,
+    noEnums: true,
+    defaultToken: "cxx::TokenKind::T_IDENTIFIER",
+    classifier: "classify",
+  });
 }

src/parser/CMakeLists.txt

@@ -69,7 +69,6 @@ add_library(cxx-parser
     cxx/windows_toolchain.cc
 
     # generated files
-    keywords-priv.h
     pp_keywords-priv.h
     # headers
     ${CXX_INCLUDE_HEADER_FILES}
@@ -97,11 +96,6 @@ target_link_libraries(cxx-parser
     PUBLIC  $<BUILD_INTERFACE:utf8cpp>
 )
 
-add_custom_command(OUTPUT keywords-priv.h
-    COMMAND kwgen < ${CMAKE_CURRENT_SOURCE_DIR}/cxx/keywords.kwgen > keywords-priv.h
-    DEPENDS cxx/keywords.kwgen
-    COMMENT "Generate keywords-priv.h")
-
 add_custom_command(OUTPUT pp_keywords-priv.h
     COMMAND kwgen < ${CMAKE_CURRENT_SOURCE_DIR}/cxx/pp_keywords.kwgen > pp_keywords-priv.h
     DEPENDS cxx/pp_keywords.kwgen

src/parser/cxx/keywords.kwgen

@@ -19,14 +19,13 @@
 // SOFTWARE.
 
 #include <cxx/lexer.h>
+#include <cxx/private/keywords-priv.h>
 #include <utf8/unchecked.h>
 
 #include <cctype>
 #include <unordered_map>
 #include <vector>
 
-#include "keywords-priv.h"
-
 namespace cxx {
 
 inline namespace {