Merge pull request #86 from yokenzan/feature/master-csv-loading-optimization

perf(master): optimize loading and search caching

Author: yokenzan

.gitignore

@@ -9,3 +9,10 @@
 /vendor/
 Gemfile.lock
 /node_modules
+/sorbet/
+/bin/tapioca
+/.serena/
+/.vscode/
+/csv/master/*/utf8/
+/csv/master/*/.cache/
+

Rakefile

@@ -1,10 +1,33 @@
 # frozen_string_literal: true
 
 require 'bundler/gem_tasks'
+require 'fileutils'
 require 'rspec/core/rake_task'
 require 'rubocop/rake_task'
 
 RSpec::Core::RakeTask.new(:test)
 RuboCop::RakeTask.new(:lint)
 
 task default: :test
+
+namespace :master do
+  desc 'Shift_JISのマスタCSV/TXTをUTF-8へ変換して保存する'
+  task :convert_utf8 do
+    master_root = File.expand_path('csv/master', __dir__)
+    year_dirs = Dir.glob(File.join(master_root, '*')).select { | path | File.directory?(path) }
+
+    year_dirs.each do | year_dir |
+      utf8_dir = File.join(year_dir, 'utf8')
+      FileUtils.mkdir_p(utf8_dir)
+
+      source_files = Dir.glob(File.join(year_dir, '*.{csv,txt,CSV,TXT}'))
+      source_files.each do | source_path |
+        source = File.binread(source_path).force_encoding('Shift_JIS')
+        utf8 = source.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        output_path = File.join(utf8_dir, File.basename(source_path))
+        File.binwrite(output_path, utf8)
+        puts "converted: #{source_path} -> #{output_path}"
+      end
+    end
+  end
+end

benchmark/master_loading.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+require 'logger'
+require 'open3'
+require 'pathname'
+require_relative '../lib/receiptisan'
+
+# rubocop:disable Metrics/ModuleLength
+module MasterLoadingBenchmark
+  VERSION = Receiptisan::Model::ReceiptComputer::Master::Version::V2024_R06
+  MASTER = Receiptisan::Model::ReceiptComputer::Master
+  LOADER_TYPES = %i[
+    shinryou_koui
+    iyakuhin
+    tokutei_kizai
+    comment
+    shoubyoumei
+    shuushokugo
+  ].freeze
+
+  class ForeachCollector
+    Result = Struct.new(:loader, :io_and_encoding, :string_processing, :object_building, :lines, keyword_init: true)
+
+    def initialize
+      @results = Hash.new do | hash, key |
+        hash[key] = Result.new(
+          loader:            key,
+          io_and_encoding:   0.0,
+          string_processing: 0.0,
+          object_building:   0.0,
+          lines:             0
+        )
+      end
+    end
+
+    def record(loader_name, io_and_encoding:, string_processing:, object_building:, lines:)
+      result = @results[loader_name]
+      result.io_and_encoding += io_and_encoding
+      result.string_processing += string_processing
+      result.object_building += object_building
+      result.lines += lines
+    end
+
+    def to_a
+      @results.values.sort_by(&:loader)
+    end
+  end
+
+  module ForeachProfiler
+    class << self
+      attr_accessor :collector
+    end
+  end
+
+  module LoaderTraitInstrument
+    def foreach(csv_paths)
+      logger.info 'prepare to load following CSV %d files:' % csv_paths.length
+      logger.info csv_paths.map(&:to_path)
+
+      io_and_encoding = 0.0
+      string_processing = 0.0
+      object_building = 0.0
+      lines = 0
+
+      csv_paths.each do | csv_path |
+        load_path, read_encoding = resolve_load_path(csv_path)
+        contents = nil
+        io_and_encoding += Benchmark.realtime do
+          contents = File.read(load_path, mode: "r:#{read_encoding}:UTF-8")
+        end
+
+        rows = contents.split("\n")
+        rows.each do | row |
+          values = nil
+          string_processing += Benchmark.realtime do
+            values = row.delete_suffix("\r").tr('"', '').split(',')
+          end
+          object_building += Benchmark.realtime do
+            yield values
+          end
+          lines += 1
+        end
+
+        logger.info "#{load_path}(#{rows.length} lines) was loaded."
+      end
+
+      ForeachProfiler.collector&.record(
+        self.class.name.split('::').last,
+        io_and_encoding:   io_and_encoding,
+        string_processing: string_processing,
+        object_building:   object_building,
+        lines:             lines
+      )
+    end
+  end
+
+  module_function
+
+  def run
+    Receiptisan::Model::ReceiptComputer::Master::Loader::LoaderTrait.prepend(LoaderTraitInstrument)
+    puts '=== Master CSV Loading Benchmark ==='
+    puts "Version: #{VERSION}"
+    puts
+
+    benchmark_full_loading
+    benchmark_by_loader_type
+    benchmark_search_command
+  end
+
+  def benchmark_full_loading
+    puts '[1] Full load benchmark (Loader#load)'
+    times = []
+    rss_diffs = []
+
+    3.times do | i |
+      before_rss = rss_kb
+      elapsed = Benchmark.realtime { build_loader.load(VERSION) }
+      after_rss = rss_kb
+      times << elapsed
+      rss_diffs << (after_rss - before_rss)
+      puts format('  run%-2d: %.3fs (RSS %+d KB)', i + 1, elapsed, after_rss - before_rss)
+    end
+
+    puts format('  cold run: %.3fs', times.first)
+    puts format('  warm avg: %.3fs', average(times.drop(1)))
+    puts format('  RSS diff avg: %.1f KB', average(rss_diffs))
+    puts
+  end
+
+  def benchmark_by_loader_type
+    puts '[2][3] Breakdown by loader type and foreach phases'
+    LOADER_TYPES.each do | type |
+      elapsed_times = []
+      collector = ForeachCollector.new
+
+      3.times do | i |
+        ForeachProfiler.collector = collector
+        elapsed = Benchmark.realtime { build_loader.load_type(VERSION, type) }
+        elapsed_times << elapsed
+        puts format('  %-15s run%-2d: %.3fs', type, i + 1, elapsed)
+      end
+
+      puts format('  %-15s cold: %.3fs / warm avg: %.3fs', type, elapsed_times.first, average(elapsed_times.drop(1)))
+      collector.to_a.each do | result |
+        puts format(
+          '    %-18s io+enc: %.3fs | split: %.3fs | build: %.3fs | lines: %d',
+          result.loader,
+          result.io_and_encoding,
+          result.string_processing,
+          result.object_building,
+          result.lines
+        )
+      end
+    ensure
+      ForeachProfiler.collector = nil
+    end
+    puts
+  end
+
+  def benchmark_search_command
+    puts '[4][5] receiptisan search end-to-end'
+    cases = [
+      ['max-shinryou', %w[--type shinryou-koui --name 初診 --month 202406]],
+      ['mid-iyakuhin', %w[--type iyakuhin --name アセト --month 202406]],
+      ['small-kizai', %w[--type tokutei-kizai --name カテーテル --month 202406]],
+    ]
+
+    cases.each do | name, args |
+      times = []
+      3.times do | i |
+        elapsed = Benchmark.realtime do
+          stdout, stderr, status = Open3.capture3(
+            'bundle', 'exec', 'ruby', 'exe/receiptisan', 'search', *args,
+            chdir: repo_root
+          )
+          next if status.success?
+
+          raise "search failed (#{name}): #{stderr}\n#{stdout}"
+        end
+        times << elapsed
+        puts format('  %-12s run%-2d: %.3fs', name, i + 1, elapsed)
+      end
+
+      puts format('  %-12s cold: %.3fs / warm avg: %.3fs', name, times.first, average(times.drop(1)))
+    end
+    puts
+  end
+
+  def build_loader
+    MASTER::Loader.new(MASTER::ResourceResolver.new, Logger.new(nil))
+  end
+
+  def rss_kb
+    line = File.read('/proc/self/status').each_line.find { | l | l.start_with?('VmRSS:') }
+    return 0 unless line
+
+    line.split[1].to_i
+  end
+
+  def average(values)
+    return 0.0 if values.empty?
+
+    values.sum(0.0) / values.length
+  end
+
+  def repo_root
+    Pathname(__dir__).join('..').expand_path.to_path
+  end
+  # rubocop:enable Metrics/ModuleLength
+end
+
+MasterLoadingBenchmark.run if $PROGRAM_NAME == __FILE__

lib/receiptisan/model/receipt_computer/master/loader.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'fileutils'
+require 'pathname'
+
 require_relative 'loader/loader_trait'
 require_relative 'loader/shinryou_koui_loader'
 require_relative 'loader/iyakuhin_loader'
@@ -31,8 +34,12 @@ def load(version)
             logger.info("preparing to load master version #{version.year}")
 
             csv_paths = @resource_resolver.detect_csv_files(version)
+            cache_path = detect_cache_path(csv_paths)
+            cache = load_from_cache(cache_path, csv_paths)
+            return cache if cache
 
-            load_from_version_and_csv(version, **csv_paths).tap do
+            load_from_version_and_csv(version, **csv_paths).tap do | master |
+              write_cache(cache_path, master)
               logger.info("loading master version #{version.year} completed")
             end
           end
@@ -71,7 +78,20 @@ def load_from_version_and_csv(
           # @return [Hash]
           def load_type(version, type)
             csv_paths = @resource_resolver.detect_csv_files(version)
+            cache_path = detect_type_cache_path(csv_paths, type)
+            cache = load_from_cache(cache_path, csv_paths)
+            return cache if cache
+
+            load_type_from_csv_paths(version, type, csv_paths).tap do | loaded |
+              write_cache(cache_path, loaded)
+            end
+          end
 
+          # @param version [Version]
+          # @param type [Symbol]
+          # @param csv_paths [Hash<Symbol, Array<Pathname>>]
+          # @return [Hash]
+          def load_type_from_csv_paths(version, type, csv_paths)
             case type
             when :shinryou_koui
               @shinryou_koui_loader.load(version, csv_paths[:shinryou_koui_csv_path])
@@ -92,6 +112,67 @@ def load_type(version, type)
 
           private
 
+          # @param csv_paths [Hash<Symbol, Array<Pathname>>]
+          # @return [Pathname]
+          def detect_cache_path(csv_paths)
+            sample_path = csv_paths.values.flatten.first
+            sample_path.parent.join('.cache', 'master.marshal')
+          end
+
+          # @param csv_paths [Hash<Symbol, Array<Pathname>>]
+          # @param type [Symbol]
+          # @return [Pathname]
+          def detect_type_cache_path(csv_paths, type)
+            sample_path = csv_paths.values.flatten.first
+            sample_path.parent.join('.cache', "#{type}.marshal")
+          end
+
+          # @param cache_path [Pathname]
+          # @param csv_paths [Hash<Symbol, Array<Pathname>>]
+          # @return [Master, nil]
+          def load_from_cache(cache_path, csv_paths)
+            return nil unless cache_available?(cache_path, csv_paths)
+
+            logger.info("loading master cache: #{cache_path}")
+            # rubocop:disable Security/MarshalLoad
+            Marshal.load(cache_path.binread)
+            # rubocop:enable Security/MarshalLoad
+          rescue StandardError => e
+            logger.warn("failed to load cache(#{cache_path}): #{e.class}: #{e.message}")
+            nil
+          end
+
+          # @param cache_path [Pathname]
+          # @param master [Master]
+          # @return [void]
+          def write_cache(cache_path, master)
+            FileUtils.mkdir_p(cache_path.dirname)
+            cache_path.binwrite(Marshal.dump(master))
+          rescue StandardError => e
+            logger.warn("failed to write cache(#{cache_path}): #{e.class}: #{e.message}")
+          end
+
+          # @param cache_path [Pathname]
+          # @param csv_paths [Hash<Symbol, Array<Pathname>>]
+          # @return [Boolean]
+          def cache_available?(cache_path, csv_paths)
+            return false unless cache_path.exist?
+
+            cache_mtime = cache_path.mtime
+            target_dir = csv_paths.values.flatten.first.parent
+            latest_mtime = target_source_paths(target_dir).map(&:mtime).max
+            return true unless latest_mtime
+
+            cache_mtime >= latest_mtime
+          end
+
+          # @param target_dir [Pathname]
+          # @return [Array<Pathname>]
+          def target_source_paths(target_dir)
+            pattern = target_dir.join('**', '*.{csv,txt,CSV,TXT}').to_path
+            Dir.glob(pattern).map { | path | Pathname(path) }
+          end
+
           attr_reader :logger
         end
       end