include token ids in release header

Author: iliabylich

.depend

@@ -2,9 +2,9 @@
 alloc.o: alloc.c alloc.h
 
 api.o: api.c api.h parser_options.h string.h decoder.h bytes.h \
-  token_rewriter.h token.h loc.h shared_byte_list.h parser_result.h \
-  source_line.h nodes.h diagnostic.h messages.h comment.h \
-  magic_comment.h test_helper.h
+  token_rewriter.h token.h loc.h token_ids.h shared_byte_list.h \
+  parser_result.h source_line.h nodes.h diagnostic.h messages.h \
+  comment.h magic_comment.h test_helper.h
 
 bytes.o: bytes.c bytes.h test_helper.h
 
@@ -24,12 +24,12 @@ messages.o: messages.c messages.h string.h test_helper.h
 nodes.o: nodes.c nodes.h string.h loc.h bytes.h test_helper.h
 
 parser_options.o: parser_options.c parser_options.h string.h decoder.h \
-  bytes.h token_rewriter.h token.h loc.h shared_byte_list.h \
+  bytes.h token_rewriter.h token.h loc.h token_ids.h shared_byte_list.h \
   test_helper.h
 
 parser_result.o: parser_result.c parser_result.h string.h source_line.h \
-  bytes.h nodes.h loc.h token.h diagnostic.h messages.h comment.h \
-  magic_comment.h test_helper.h
+  bytes.h nodes.h loc.h token.h token_ids.h diagnostic.h messages.h \
+  comment.h magic_comment.h test_helper.h
 
 shared_byte_list.o: shared_byte_list.c shared_byte_list.h test_helper.h
 
@@ -39,17 +39,17 @@ string.o: string.c string.h test_helper.h
 
 test_helper.o: test_helper.c test_helper.h string.h
 
-token.o: token.c token.h bytes.h loc.h test_helper.h
+token.o: token.c token.h bytes.h loc.h token_ids.h test_helper.h
 
 token_rewriter.o: token_rewriter.c token_rewriter.h token.h bytes.h loc.h \
-  shared_byte_list.h test_helper.h
+  token_ids.h shared_byte_list.h test_helper.h
 
 tests/runner: tests/test.c tests/../test_helper.h tests/../bytes.h \
   tests/../comment.h tests/../loc.h tests/../decoder.h tests/../string.h \
   tests/../diagnostic.h tests/../messages.h tests/../magic_comment.h \
   tests/../nodes.h tests/../parser_options.h tests/../token_rewriter.h \
-  tests/../token.h tests/../shared_byte_list.h tests/../parser_result.h \
-  tests/../source_line.h tests/../api.h
+  tests/../token.h tests/../token_ids.h tests/../shared_byte_list.h \
+  tests/../parser_result.h tests/../source_line.h tests/../api.h
 
 benchmark/c-parser: benchmark/benchmark.c benchmark/../lib-ruby-parser.h \
   benchmark/rb_filelist.h

codegen/Cargo.toml

@@ -5,6 +5,8 @@ version = "0.1.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[build-dependencies]
+[dependencies]
 cbindgen = "0.20.0"
+
+[build-dependencies]
 lib-ruby-parser-nodes = {version = "0.38.0"}

codegen/build.mk

@@ -38,9 +38,19 @@ ruby-parser-c/src/message.rs: $(CODEGEN_DEPS)
 	$(DO_CODEGEN)
 CLEAN += ruby-parser-c/src/message.rs
 
-lib-ruby-parser.h: codegen/examples/merge_headers.rs $(H_FILES)
+# token_id.{h,c} codegen
+DO_CODEGEN_TOKEN_IDS = cargo run --example build_token_ids --manifest-path codegen/Cargo.toml
+
+token_ids.h: codegen/examples/build_token_ids.rs
+	$(DO_CODEGEN_TOKEN_IDS)
+CLEAN += token_ids.h
+update-depend: token_ids.h
+
+# lib-ruby-parser.h codegen
+lib-ruby-parser.h: codegen/examples/merge_headers.rs $(H_FILES) token_ids.h
 	cargo run --example merge_headers --manifest-path codegen/Cargo.toml
 CLEAN += lib-ruby-parser.h
+update-depend: lib-ruby-parser.h
 
 # manual codegen task
 do-codegen:

codegen/build.rs

@@ -1,54 +1,7 @@
-extern crate cbindgen;
 extern crate lib_ruby_parser_nodes;
 
 mod codegen;
 
 fn main() {
     codegen::codegen();
-
-    tokend_ids::codegen();
-}
-
-mod tokend_ids {
-    const TOKEN_IDS: &str = "../token_ids.h";
-
-    pub(crate) fn codegen() {
-        cbindgen::Builder::new()
-            .with_crate("../ruby-parser-c")
-            .with_parse_deps(true)
-            .with_parse_extra_bindings(&["lib-ruby-parser"])
-            .with_no_includes()
-            .include_item("Lexer")
-            .generate()
-            .expect("Unable to generate bindings")
-            .write_to_file(TOKEN_IDS);
-
-        let contents = std::fs::read_to_string(TOKEN_IDS).unwrap();
-        let mut token_ids = vec![];
-        for line in contents.lines() {
-            if line.starts_with("/// Token") {
-                // token comment line
-                token_ids.push(line.to_string());
-            } else if line.contains("const int32_t Lexer_") {
-                let line = line
-                    .replace("static ", "")
-                    .replace("Lexer_", "LIB_RUBY_PARSER_TOKEN_");
-                token_ids.push(line);
-                token_ids.push(String::from(""))
-            }
-        }
-
-        let contents = format!(
-            "#ifndef LIB_RUBY_PARSER_TOKEN_IDS_H
-#define LIB_RUBY_PARSER_TOKEN_IDS_H
-
-#include <stdint.h>
-
-{token_ids}
-#endif // LIB_RUBY_PARSER_TOKEN_IDS_H",
-            token_ids = token_ids.join("\n")
-        );
-
-        std::fs::write(TOKEN_IDS, contents).unwrap();
-    }
 }

codegen/examples/build_token_ids.rs

@@ -0,0 +1,88 @@
+extern crate cbindgen;
+
+#[derive(Debug, Default)]
+struct TokenId {
+    comment: String,
+    name: String,
+    value: String,
+}
+
+impl TokenId {
+    fn declaration(&self) -> String {
+        format!(
+            "{comment}
+#define LIB_RUBY_PARSER_TOKEN_{name} {value}",
+            comment = self.comment,
+            name = self.name,
+            value = self.value
+        )
+    }
+}
+
+fn bindgen() -> String {
+    const TMP_BINDINGS: &str = "codegen/examples/target/tmp_bindings.h";
+
+    cbindgen::Builder::new()
+        .with_crate("ruby-parser-c")
+        .with_parse_deps(true)
+        .with_parse_extra_bindings(&["lib-ruby-parser"])
+        .with_no_includes()
+        .include_item("Lexer")
+        .generate()
+        .expect("Unable to generate bindings")
+        .write_to_file(TMP_BINDINGS);
+
+    let contents = std::fs::read_to_string(TMP_BINDINGS).unwrap();
+
+    std::fs::remove_file(TMP_BINDINGS).unwrap();
+
+    contents
+}
+
+fn collect_token_ids() -> Vec<TokenId> {
+    let bindings = bindgen();
+
+    let mut token_ids = vec![];
+    let mut token_id = TokenId::default();
+
+    for line in bindings.lines() {
+        if line.starts_with("/// Token") {
+            // token comment line
+            token_id.comment = line.to_string();
+        } else if line.starts_with("static const int32_t Lexer_") {
+            let line = line
+                .replace("static const int32_t Lexer_", "")
+                .replace(";", "");
+
+            let parts = line.split(" = ").collect::<Vec<_>>();
+            assert_eq!(parts.len(), 2);
+            token_id.name = parts[0].to_string();
+            token_id.value = parts[1].to_string();
+
+            token_ids.push(std::mem::take(&mut token_id));
+        }
+    }
+
+    token_ids
+}
+
+pub fn main() {
+    let token_ids = collect_token_ids();
+
+    let token_ids_h = format!(
+        "#ifndef LIB_RUBY_PARSER_TOKEN_IDS_H
+#define LIB_RUBY_PARSER_TOKEN_IDS_H
+
+{declarations}
+
+#endif // LIB_RUBY_PARSER_TOKEN_IDS_H
+",
+        declarations = token_ids
+            .iter()
+            .map(TokenId::declaration)
+            .collect::<Vec<_>>()
+            .join("\n\n")
+    );
+
+    std::fs::write("token_ids.h", token_ids_h).unwrap();
+}

codegen/examples/merge_headers.rs

@@ -4,6 +4,7 @@ const HEADERS: &[&str] = &[
     "bytes.h",
     "source_line.h",
     "loc.h",
+    "token_ids.h",
     "token.h",
     "comment.h",
     "magic_comment.h",
@@ -49,7 +50,7 @@ fn process_header(header_path: &str, sys_includes: &mut Vec<String>, contents: &
             continue;
         }
         if line.starts_with("#ifndef LIB_RUBY_PARSER_")
-            || line.starts_with("#define LIB_RUBY_PARSER_")
+            || (line.starts_with("#define LIB_RUBY_PARSER_") && line.ends_with("_H"))
             || line.starts_with("#endif // LIB_RUBY_PARSER_")
         {
             // header guard, skip

token.c

@@ -13,6 +13,7 @@ static void test_token_fields(void)
 
     LIB_RUBY_PARSER_Token token = lib_ruby_parser__test__make_token_eq((LIB_RUBY_PARSER_Loc){.begin = 1, .end = 2}, 3, 4);
 
+    assert_eq(token.token_type, LIB_RUBY_PARSER_TOKEN_tEQ);
     assert_token(token, "tEQ");
     assert_byte_list(token.token_value.raw, "=");
     assert_loc(token.loc, 1, 2);

token.h

@@ -3,6 +3,7 @@
 
 #include "bytes.h"
 #include "loc.h"
+#include "token_ids.h"
 
 /*
     Equivalent of `lib_ruby_parser::Token`