switch to a new codegen executable

Author: iliabylich

.depend

@@ -5,71 +5,71 @@ api.o: api.cpp api.hpp parser_options.hpp string.hpp decoder.hpp \
   bytes.hpp token_rewriter.hpp token.hpp loc.hpp decoded_input.hpp \
   source_line.hpp token_ids.hpp shared_byte_list.hpp parser_result.hpp \
   nodes.hpp diagnostic.hpp messages.hpp comment.hpp magic_comment.hpp \
-  utils.hpp test_helper.hpp
+  utils.hpp
 
-bytes.o: bytes.cpp bytes.hpp utils.hpp test_helper.hpp
+bytes.o: bytes.cpp bytes.hpp utils.hpp
 
 comment.o: comment.cpp utils.hpp comment.hpp loc.hpp decoded_input.hpp \
-  string.hpp source_line.hpp bytes.hpp test_helper.hpp
+  string.hpp source_line.hpp bytes.hpp
 
 decoded_input.o: decoded_input.cpp utils.hpp decoded_input.hpp string.hpp \
-  source_line.hpp bytes.hpp test_helper.hpp
+  source_line.hpp bytes.hpp
 
-decoder.o: decoder.cpp decoder.hpp string.hpp bytes.hpp utils.hpp \
-  test_helper.hpp
+decoder.o: decoder.cpp decoder.hpp string.hpp bytes.hpp utils.hpp
 
 diagnostic.o: diagnostic.cpp diagnostic.hpp messages.hpp string.hpp \
-  loc.hpp decoded_input.hpp source_line.hpp bytes.hpp utils.hpp \
-  test_helper.hpp
+  loc.hpp decoded_input.hpp source_line.hpp bytes.hpp utils.hpp
 
 loc.o: loc.cpp utils.hpp loc.hpp decoded_input.hpp string.hpp \
-  source_line.hpp bytes.hpp test_helper.hpp
+  source_line.hpp bytes.hpp
 
 magic_comment.o: magic_comment.cpp magic_comment.hpp loc.hpp \
-  decoded_input.hpp string.hpp source_line.hpp bytes.hpp utils.hpp \
-  test_helper.hpp
+  decoded_input.hpp string.hpp source_line.hpp bytes.hpp utils.hpp
 
-messages.o: messages.cpp messages.hpp string.hpp utils.hpp \
-  test_helper.hpp
+messages.o: messages.cpp messages.hpp string.hpp utils.hpp
 
 nodes.o: nodes.cpp nodes.hpp string.hpp loc.hpp decoded_input.hpp \
-  source_line.hpp bytes.hpp utils.hpp test_helper.hpp
+  source_line.hpp bytes.hpp utils.hpp
 
 parser_options.o: parser_options.cpp parser_options.hpp string.hpp \
   decoder.hpp bytes.hpp token_rewriter.hpp token.hpp loc.hpp \
   decoded_input.hpp source_line.hpp token_ids.hpp shared_byte_list.hpp \
-  utils.hpp test_helper.hpp
+  utils.hpp
 
 parser_result.o: parser_result.cpp parser_result.hpp nodes.hpp string.hpp \
   loc.hpp decoded_input.hpp source_line.hpp bytes.hpp token.hpp \
   token_ids.hpp diagnostic.hpp messages.hpp comment.hpp \
-  magic_comment.hpp utils.hpp test_helper.hpp
+  magic_comment.hpp utils.hpp
 
-shared_byte_list.o: shared_byte_list.cpp shared_byte_list.hpp \
-  test_helper.hpp
+shared_byte_list.o: shared_byte_list.cpp shared_byte_list.hpp
 
-source_line.o: source_line.cpp source_line.hpp utils.hpp test_helper.hpp
+source_line.o: source_line.cpp source_line.hpp utils.hpp
 
-string.o: string.cpp string.hpp utils.hpp test_helper.hpp
-
-test_helper.o: test_helper.cpp test_helper.hpp
+string.o: string.cpp string.hpp utils.hpp
 
 token.o: token.cpp token.hpp bytes.hpp loc.hpp decoded_input.hpp \
-  string.hpp source_line.hpp token_ids.hpp utils.hpp test_helper.hpp
+  string.hpp source_line.hpp token_ids.hpp utils.hpp
 
 token_rewriter.o: token_rewriter.cpp token_rewriter.hpp token.hpp \
   bytes.hpp loc.hpp decoded_input.hpp string.hpp source_line.hpp \
-  token_ids.hpp shared_byte_list.hpp utils.hpp test_helper.hpp
-
-tests/runner: tests/test.cpp tests/../test_helper.hpp tests/../bytes.hpp \
-  tests/../comment.hpp tests/../loc.hpp tests/../decoded_input.hpp \
-  tests/../string.hpp tests/../source_line.hpp tests/../decoder.hpp \
-  tests/../diagnostic.hpp tests/../messages.hpp \
-  tests/../magic_comment.hpp tests/../nodes.hpp \
-  tests/../parser_options.hpp tests/../token_rewriter.hpp \
-  tests/../token.hpp tests/../token_ids.hpp \
-  tests/../shared_byte_list.hpp tests/../parser_result.hpp \
-  tests/../api.hpp
+  token_ids.hpp shared_byte_list.hpp utils.hpp
+
+tests/runner: tests/runner.cpp tests/test_helper.hpp \
+  tests/test_helper.cpp tests/bytes_test.cpp bytes.hpp \
+  utils.hpp tests/comment_test.cpp comment.hpp \
+  loc.hpp decoded_input.hpp string.hpp \
+  source_line.hpp tests/decoded_input_test.cpp \
+  tests/decoder_test.cpp decoder.hpp tests/diagnostic_test.cpp \
+  diagnostic.hpp messages.hpp tests/loc_test.cpp \
+  tests/magic_comment_test.cpp magic_comment.hpp \
+  tests/messages_test.cpp tests/nodes_test.cpp nodes.hpp \
+  tests/parser_options_test.cpp parser_options.hpp \
+  token_rewriter.hpp token.hpp token_ids.hpp \
+  shared_byte_list.hpp tests/parser_result_test.cpp \
+  parser_result.hpp tests/shared_byte_list_test.cpp \
+  tests/source_line_test.cpp tests/string_test.cpp \
+  tests/token_rewriter_test.cpp tests/token_test.cpp tests/api_test.cpp \
+  api.hpp
 
 benchmark/cpp-parser: benchmark/benchmark.cpp benchmark/rb_filelist.hpp \
   benchmark/../lib-ruby-parser.hpp

.github/workflows/release.yml

@@ -57,11 +57,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: codegen
-        env:
-          RUSTFLAGS: ""
-        run: |
-          make do-codegen
-          make token_ids.hpp
+        run: make do-codegen
 
       - name: make lib file
         run: |

.github/workflows/test.yml

@@ -87,11 +87,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: codegen
-        env:
-          RUSTFLAGS: ""
-        run: |
-          make do-codegen
-          make token_ids.hpp
+        run: make do-codegen
 
       - name: run tests
         run: ${{ matrix.build.run }}

.gitignore

@@ -9,6 +9,8 @@ messages.cpp
 nodes.hpp
 nodes.cpp
 token_ids.hpp
+tests/messages_test.cpp
+tests/nodes_test.cpp
 target
 *.o
 *.a
@@ -23,3 +25,5 @@ benchmark/filelist
 benchmark/stats
 *.benchmark-out
 docs/
+codegen/codegen
+codegen/merge-headers

Makefile

@@ -19,15 +19,12 @@ SOURCES = \
 	shared_byte_list \
 	source_line \
 	string \
-	token \
-	token_rewriter
-
-ifeq ($(BUILD_ENV), debug)
-SOURCES += test_helper
-endif
+	token_rewriter \
+	token
 
 CPP_FILES = $(foreach source,$(SOURCES),$(source).cpp)
 HPP_FILES = $(foreach source,$(SOURCES),$(source).hpp)
+HPP_FILES += token_ids.hpp
 O_FILES = $(foreach source,$(SOURCES),$(source).$(O))
 STATIC_LIB = libruby_parser_cpp.$(A)
 
@@ -64,6 +61,6 @@ clean:
 	rm -f *.$(A)
 	rm -rf *.dSYM
 
-doxygen: messages.hpp nodes.hpp token_ids.hpp
+doxygen: $(CODEGEN_FILES)
 	rm -rf docs
 	doxygen

codegen/Cargo.toml

@@ -1,48 +1,85 @@
-CODEGEN_DEPS = $(wildcard codegen/codegen/*.rs)
-CODEGEN_DEPS += codegen/Cargo.toml
-CODEGEN_DEPS += codegen/build.rs
+define download_file
+URL="$(1)" SAVE_AS="$(2)" ruby scripts/download_file.rb
+endef
 
-DO_CODEGEN = cd codegen && touch build.rs && cargo build
+CODEGEN_EXE = codegen/codegen$(EXE)
+$(CODEGEN_EXE):
+	$(call download_file, https://github.com/lib-ruby-parser/lib-ruby-parser/releases/download/v4.0.2+ruby-3.1.1/codegen-$(TARGET)$(EXE), $@)
+	chmod +x $(CODEGEN_EXE)
+CLEAN += $(CODEGEN_EXE)
 
 # Codegen deps
-messages.cpp: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
-CLEAN += messages.cpp
+tests/messages_test.cpp: codegen/messages_test.cpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
+CLEAN += tests/messages_test.cpp
+CODEGEN_FILES += tests/messages_test.cpp
 
-messages.hpp: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+messages.hpp: codegen/messages.hpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += messages.hpp
+CODEGEN_FILES += messages.hpp
 
-nodes.cpp: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
-CLEAN += nodes.cpp
+messages.cpp: codegen/messages.cpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
+CLEAN += messages.cpp
+CODEGEN_FILES += messages.cpp
 
-nodes.hpp: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+tests/nodes_test.cpp: codegen/nodes_test.cpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
+CLEAN += tests/nodes_test.cpp
+CODEGEN_FILES += tests/nodes_test.cpp
+
+nodes.hpp: codegen/nodes.hpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += nodes.hpp
+CODEGEN_FILES += nodes.hpp
+
+nodes.cpp: codegen/nodes.cpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
+CLEAN += nodes.cpp
+CODEGEN_FILES += nodes.cpp
 
-ruby-parser-cpp/src/node.rs: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+ruby-parser-cpp/src/node.rs: codegen/nodes.rs.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += ruby-parser-cpp/src/node.rs
+CODEGEN_FILES += ruby-parser-cpp/src/node.rs
 
-ruby-parser-cpp/src/message.rs: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+ruby-parser-cpp/src/message.rs: codegen/messages.rs.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += ruby-parser-cpp/src/message.rs
+CODEGEN_FILES += ruby-parser-cpp/src/message.rs
 
-# token_id.hpp codegen
-DO_CODEGEN_TOKEN_IDS = cargo run --example build_token_ids --manifest-path codegen/Cargo.toml
-
-token_ids.hpp: codegen/examples/build_token_ids.rs
-	$(DO_CODEGEN_TOKEN_IDS)
+token_ids.hpp: codegen/token_ids.hpp.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += token_ids.hpp
-update-depend: token_ids.hpp
+CODEGEN_FILES += token_ids.hpp
+
+codegen/merge-headers:
+	wget -q https://github.com/iliabylich/merge_headers/releases/download/v1.0.0/merge-headers-$(TARGET) -O codegen/merge-headers
+	chmod +x codegen/merge-headers
+CLEAN += codegen/merge-headers
+
+# A literal space.
+space :=
+space +=
+
+# Joins elements of the list in arg 2 with the given separator.
+#   1. Element separator.
+#   2. The list.
+join-with = $(subst $(space),$1,$(strip $2))
 
 # lib-ruby-parser.hpp codegen
-lib-ruby-parser.hpp: codegen/examples/merge_headers.rs $(HPP_FILES) token_ids.hpp
-	cargo run --example merge_headers --manifest-path codegen/Cargo.toml
+lib-ruby-parser.hpp: codegen/merge-headers $(HPP_FILES)
+	./codegen/merge-headers \
+		--cc $(CXX) \
+		--headers "$(call join-with,;,$(HPP_FILES))" \
+		--write-to lib-ruby-parser.hpp \
+		--include-guard-prefix LIB_RUBY_PARSER_ \
+		--output-guard LIB_RUBY_PARSER_H
+
 CLEAN += lib-ruby-parser.hpp
 update-depend: lib-ruby-parser.hpp
 
-# manual codegen task
-do-codegen:
-	$(DO_CODEGEN)
+do-codegen: $(CODEGEN_FILES)
+
+.PHONY: do-codegen