Commit generated files (ruby#1410)

Closes ruby#1107 and will likely prevent
issues like https://bugs.ruby-lang.org/issues/21541 in the future

With ignoring generated files in the source control, we always need one
extra step to make the gem usable, which limits how the gem can be
distributed (ruby#1107) and make post-distribution issues easier to happen
(which are in general harder to fix too).

If gems like ~~`prism` and~~ `rbs` and `debug` can avoid this issue by
committing generated files and verifying them on CI, I think we should
do that too.

Author: st0012

.github/workflows/lint.yml

@@ -27,6 +27,3 @@ jobs:
         bundler-cache: true
     - name: Run rubocop
       run: bundle exec rubocop
-    - name: Sanity check for the format_generated_files task
-      run: bundle exec rake generate format_generated_files
-

.github/workflows/test.yml

@@ -50,6 +50,8 @@ jobs:
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true # 'bundle install' and cache
+    - name: Verify generated parser files
+      run: bundle exec rake verify_generated
     - name: Run test
       run: bundle exec rake
       env:

.gitignore

@@ -8,10 +8,6 @@
 /TAGS
 /html
 /_site
-/lib/rdoc/rd/block_parser.rb
-/lib/rdoc/rd/inline_parser.rb
-/lib/rdoc/markdown.rb
-/lib/rdoc/markdown/literals.rb
 /pkg
 /tmp
 Gemfile.lock

Gemfile

@@ -3,7 +3,7 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake'
-gem 'racc', '> 1.4.10'
+gem 'racc', '1.8.1'
 gem 'kpeg', '>= 1.3.3'
 gem 'test-unit'
 gem 'test-unit-ruby-core'

Rakefile

@@ -5,6 +5,7 @@ $:.unshift File.expand_path('lib', __dir__) # default template dir
 require_relative 'lib/rdoc/task'
 require 'bundler/gem_tasks'
 require 'rake/testtask'
+require 'rubocop/rake_task'
 
 task :test    => [:normal_test, :rubygems_test]
 
@@ -37,63 +38,94 @@ end
 
 Rake::TestTask.new(:normal_test) do |t|
   t.verbose = true
-  t.deps = :generate
   t.test_files = FileList["test/**/*_test.rb"].exclude("test/rdoc/rdoc_rubygems_hook_test.rb")
 end
 
 Rake::TestTask.new(:rubygems_test) do |t|
   t.verbose = true
-  t.deps = :generate
   t.pattern = "test/rdoc/rdoc_rubygems_hook_test.rb"
 end
 
-path = "pkg/#{Bundler::GemHelper.gemspec.full_name}"
-
-package_parser_files = PARSER_FILES.map do |parser_file|
-  name = File.basename(parser_file, File.extname(parser_file))
-  _path = File.dirname(parser_file)
-  package_parser_file = "#{path}/#{name}.rb"
-  parsed_file = "#{_path}/#{name}.rb"
-
-  file package_parser_file => parsed_file # ensure copy runs before racc
-
-  package_parser_file
-end
-
-parsed_files = PARSER_FILES.map do |parser_file|
+def generate_parser_file(parser_file)
   ext = File.extname(parser_file)
   parsed_file = "#{parser_file.chomp(ext)}.rb"
 
-  file parsed_file => parser_file do |t|
-    puts "Generating #{parsed_file}..."
-    case ext
-    when '.ry' # need racc
-      racc = Gem.bin_path 'racc', 'racc'
-      rb_file = parser_file.gsub(/\.ry\z/, ".rb")
-      ruby "#{racc} -l -E -o #{rb_file} #{parser_file}"
-      File.open(rb_file, 'r+') do |f|
-        newtext = "# frozen_string_literal: true\n#{f.read}"
-        f.rewind
-        f.write newtext
-      end
-    when '.kpeg' # need kpeg
-      kpeg = Gem.bin_path 'kpeg', 'kpeg'
-      rb_file = parser_file.gsub(/\.kpeg\z/, ".rb")
-      ruby "#{kpeg} -fsv -o #{rb_file} #{parser_file}"
-      File.write(rb_file, File.read(rb_file).gsub(/ +$/, '')) # remove trailing spaces
+  puts "Generating #{parsed_file}..."
+  case ext
+  when '.ry' # need racc
+    sh "bundle", "exec", "racc", "-l", "-E", "-o", parsed_file, parser_file
+    File.open(parsed_file, 'r+') do |f|
+      newtext = "# frozen_string_literal: true\n#{f.read}"
+      f.rewind
+      f.write newtext
     end
+  when '.kpeg' # need kpeg
+    sh "bundle", "exec", "kpeg", "-fsv", "-o", parsed_file, parser_file
+    File.write(parsed_file, File.read(parsed_file).gsub(/ +$/, '')) # remove trailing spaces
   end
 
   parsed_file
 end
 
-task "#{path}.gem" => package_parser_files
 desc "Generate all files used racc and kpeg"
-task :generate => parsed_files
+task :generate do
+  generated_files = PARSER_FILES.map { |f| generate_parser_file(f) }
+
+  # Run RuboCop autocorrect on generated files
+  require 'rubocop'
+  cli = RuboCop::CLI.new
+  cli.run([
+    "--config=.generated_files_rubocop.yml",
+    "--autocorrect",
+    *generated_files
+  ])
+end
+
+desc "Verify that generated parser files are up to date"
+# Note: This task generates files to verify changes in the parser files.
+# So the result is not deterministic with multiple runs.
+# We can improve this by using a temporary directory and checking the diff but it's more complex to maintain.
+task :verify_generated do
+  parsed_files = PARSER_FILES.map { |f| f.sub(/\.(ry|kpeg)\z/, '.rb') }
+
+  # Save current state of generated files
+  original_content = {}
+  parsed_files.each do |file|
+    original_content[file] = File.read(file)
+  end
+
+  # Generate files from current source
+  Rake::Task[:generate].invoke
+
+  # Check if any files changed
+  changed_files = []
+  parsed_files.each do |file|
+    unless File.exist?(file)
+      abort "Generated file #{file} does not exist!"
+    end
+
+    new_content = File.read(file)
+    if original_content[file] != new_content
+      changed_files << file
+    end
+  end
+
+  if changed_files.empty?
+    puts "Generated parser files are up to date."
+  else
+    puts "Generated parser files are out of date!"
+    puts "Please run 'rake generate' to update the files."
+    puts
+    puts "Files that are out of date:"
+    changed_files.each { |f| puts "  - #{f}" }
+    exit 1
+  end
+end
 
 task :clean do
-  parsed_files.each do |path|
-    File.delete(path) if File.exist?(path)
+  PARSER_FILES.each do |parser_file|
+    parsed_file = parser_file.sub(/\.(ry|kpeg)\z/, '.rb')
+    File.delete(parsed_file) if File.exist?(parsed_file)
   end
 end
 
@@ -106,16 +138,6 @@ namespace :build do
       abort("Expected Ruby to be cloned under the same parent directory as RDoc to use this task")
     end
 
-    mv("#{path}.gem", target)
-  end
-end
-
-begin
-  require 'rubocop/rake_task'
-rescue LoadError
-else
-  RuboCop::RakeTask.new(:format_generated_files) do |t|
-    t.options = parsed_files + ["--config=.generated_files_rubocop.yml"]
+    mv("pkg/#{Bundler::GemHelper.gemspec.full_name}.gem", target)
   end
-  task :build => [:generate, "format_generated_files:autocorrect"]
 end