codegen token_ids.h with a new codegen binary that has {{ tokens }} variable now

Author: iliabylich

.github/workflows/release.yml

@@ -71,13 +71,6 @@ jobs:
       - name: checkout
         uses: actions/checkout@v2
 
-      - name: codegen
-        env:
-          RUSTFLAGS: ""
-        run: |
-          make do-codegen
-          make token_ids.h
-
       - name: make lib file
         run: |
           make ${{ matrix.build.lib_file }}

.github/workflows/test.yml

@@ -100,13 +100,6 @@ jobs:
       - name: checkout
         uses: actions/checkout@v2
 
-      - name: codegen
-        env:
-          RUSTFLAGS: ""
-        run: |
-          make do-codegen
-          make token_ids.h
-
       - name: run tests
         run: ${{ matrix.build.run }}
 

Makefile

@@ -25,6 +25,7 @@ H_FILES = alloc.h \
 	parser_result.h \
 	source_line.h \
 	token.h \
+	token_ids.h \
 	token_rewriter.h
 O_FILES = $(foreach source,$(SOURCES),$(source).$(O))
 STATIC_LIB = libruby_parser_c.$(A)

codegen/build.mk

@@ -3,9 +3,8 @@ URL="$(1)" SAVE_AS="$(2)" ruby scripts/download_file.rb
 endef
 
 CODEGEN_EXE = codegen/codegen$(EXE)
-
 $(CODEGEN_EXE):
-	$(call download_file, https://github.com/lib-ruby-parser/nodes/releases/download/v0.52.0/codegen-$(TARGET)$(EXE), $@)
+	$(call download_file, https://github.com/lib-ruby-parser/lib-ruby-parser/releases/download/v4.0.2+ruby-3.1.1/codegen-$(TARGET)$(EXE), $@)
 	chmod +x $(CODEGEN_EXE)
 CLEAN += $(CODEGEN_EXE)
 
@@ -34,13 +33,9 @@ ruby-parser-c/src/message.rs: codegen/messages.rs.liquid $(CODEGEN_EXE)
 	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += ruby-parser-c/src/message.rs
 
-# token_id.{h,c} codegen
-DO_CODEGEN_TOKEN_IDS = cargo run --example build_token_ids --manifest-path codegen/Cargo.toml
-
-token_ids.h: codegen/examples/build_token_ids.rs
-	$(DO_CODEGEN_TOKEN_IDS)
+token_ids.h: codegen/token_ids.h.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += token_ids.h
-update-depend: token_ids.h
 
 merge-headers:
 	wget -q https://github.com/iliabylich/merge_headers/releases/download/v1.0.0/merge-headers-$(TARGET) -O merge-headers
@@ -60,14 +55,10 @@ join-with = $(subst $(space),$1,$(strip $2))
 lib-ruby-parser.h: merge-headers $(H_FILES) token_ids.h nodes.h messages.h
 	./merge-headers \
 		--cc $(CC) \
-		--headers "$(call join-with,;,$(H_FILES));token_ids.h" \
+		--headers "$(call join-with,;,$(H_FILES))" \
 		--write-to lib-ruby-parser.h \
 		--include-guard-prefix LIB_RUBY_PARSER_ \
 		--output-guard LIB_RUBY_PARSER_H
 
 CLEAN += lib-ruby-parser.h
 update-depend: lib-ruby-parser.h
-
-# manual codegen task
-do-codegen:
-	$(DO_CODEGEN)

codegen/token_ids.h.liquid

@@ -0,0 +1,13 @@
+#ifndef LIB_RUBY_PARSER_TOKEN_IDS_H
+#define LIB_RUBY_PARSER_TOKEN_IDS_H
+
+/// @defgroup token_ids Token IDs
+/// @{
+/// @}
+
+{% for token in tokens %}
+/// {{ token.comment }}
+#define LIB_RUBY_PARSER_TOKEN_{{ token.name }} {{ token.value }}
+{% endfor %}
+
+#endif // LIB_RUBY_PARSER_TOKEN_IDS_H