Fix compiler output equivalence and add comprehensive tests

- Fix forloop.first/last/size lookups to try hash key before method call
- Fix tablerow cols parameter to use attributes['cols'] not @cols
- Fix tablerow output format to match interpreter (newlines, row boundaries)
- Fix capture compiler to access @Body directly instead of iterating nodelist
- Add CompiledTemplate class to encapsulate code and external_tags
- Add external filter support via filter_handler
- Add debug mode warnings for external tag/filter calls
- Add Ruby 3.3 compatibility shim for peek_byte/scan_byte
- Add comprehensive unit tests for output equivalence
- Add benchmark comparing compiled vs interpreted rendering

All 30 test templates now produce identical output between compiled
Ruby and interpreted Liquid. Pre-compiled Ruby is 1.68x faster.

Author: claude

lib/liquid/compile.rb

@@ -48,6 +48,7 @@
 #
 module Liquid
   module Compile
+    autoload :CompiledTemplate, 'liquid/compile/compiled_template'
     autoload :CodeGenerator, 'liquid/compile/code_generator'
     autoload :RubyCompiler, 'liquid/compile/ruby_compiler'
     autoload :ExpressionCompiler, 'liquid/compile/expression_compiler'

lib/liquid/compile/compiled_template.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Liquid
+  module Compile
+    # Represents a compiled Liquid template ready for execution.
+    #
+    # Contains the Ruby source code and any external tags/filters that need to be
+    # passed to the generated lambda at runtime.
+    #
+    # Usage:
+    #   compiled = template.compile_to_ruby
+    #   result = compiled.call({ "name" => "World" })
+    #
+    #   # With custom filters:
+    #   compiled.filter_handler = MyFilterModule
+    #   result = compiled.call({ "name" => "World" })
+    #
+    class CompiledTemplate
+      attr_reader :code, :external_tags
+      attr_accessor :filter_handler
+
+      # @param code [String] The generated Ruby code
+      # @param external_tags [Hash] Map of variable names to Tag objects for runtime delegation
+      # @param has_external_filters [Boolean] Whether external filters are used
+      def initialize(code, external_tags = {}, has_external_filters = false)
+        @code = code
+        @external_tags = external_tags
+        @has_external_filters = has_external_filters
+        @filter_handler = nil
+        @proc = nil
+      end
+
+      # Returns true if this template has external tags that need runtime delegation
+      def has_external_tags?
+        !@external_tags.empty?
+      end
+
+      # Returns true if this template uses external filters
+      def has_external_filters?
+        @has_external_filters
+      end
+
+      # Returns the compiled proc, caching it after first compilation
+      def to_proc
+        @proc ||= eval(@code)
+      end
+
+      # Execute the compiled template with the given assigns
+      # @param assigns [Hash] The variable assignments
+      # @param filter_handler [Object] Optional filter handler to override the default
+      # @return [String] The rendered output
+      def call(assigns = {}, filter_handler: nil)
+        proc = to_proc
+        handler = filter_handler || @filter_handler
+
+        # Build arguments based on what the lambda expects
+        args = [assigns]
+        args << @external_tags if has_external_tags?
+        args << handler if has_external_filters?
+
+        proc.call(*args)
+      end
+
+      # Returns the Ruby code as a string
+      def to_s
+        @code
+      end
+    end
+  end
+
+  # Make CompiledTemplate available at the top level for convenience
+  CompiledTemplate = Compile::CompiledTemplate
+end

lib/liquid/compile/expression_compiler.rb

@@ -68,14 +68,9 @@ def self.compile_variable_lookup(lookup, compiler)
             # Numeric index like foo[0]
             base = "__lookup__(#{base}, #{key})"
           elsif key.is_a?(String)
-            # Check if this is a command method (size, first, last)
-            if lookup.lookup_command?(index)
-              # Call as method
-              base = "(#{base}.respond_to?(:#{key}) ? #{base}.#{key} : nil)"
-            else
-              # Access as hash/array key
-              base = "__lookup__(#{base}, #{key.inspect})"
-            end
+            # Always use __lookup__ which tries key access first,
+            # then falls back to method call for command methods (first, last, size)
+            base = "__lookup__(#{base}, #{key.inspect})"
           else
             base = "__lookup__(#{base}, #{compile(key, compiler)})"
           end

lib/liquid/compile/filter_compiler.rb

@@ -206,19 +206,22 @@ def self.compile_filter(input, name, args, kwargs, compiler)
       end
 
       # Compile a filter that's not built-in
+      # Uses __call_filter__ helper which must be provided by the runtime
       def self.compile_generic_filter(input, name, args, kwargs, compiler)
+        # Mark that we're using external filters
+        compiler.register_external_filter
+
         compiled_args = args.map { |arg| compile_arg(arg, compiler) }
 
         if kwargs && !kwargs.empty?
           kwargs_hash = kwargs.map { |k, v| "#{k.inspect} => #{compile_arg(v, compiler)}" }.join(", ")
           compiled_args << "{ #{kwargs_hash} }"
         end
 
-        args_str = compiled_args.empty? ? "" : ", #{compiled_args.join(', ')}"
+        args_str = compiled_args.empty? ? "[]" : "[#{compiled_args.join(', ')}]"
 
-        # Generate a method call - this assumes the filter is available as a method
-        # In practice, custom filters would need to be defined in the compiled code
-        "(respond_to?(:#{name}) ? #{name}(#{input}#{args_str}) : #{input})"
+        # Call through the filter helper which delegates to registered filters
+        "__call_filter__.call(#{name.inspect}, #{input}, #{args_str})"
       end
 
       # Compile a filter argument

lib/liquid/compile/ruby_compiler.rb

@@ -72,6 +72,35 @@ def initialize(template, options = {})
         @partials = {}        # Registered partials: name => method_name
         @partial_sources = {} # Partial sources: name => source code
         @partial_counter = 0
+        @external_tags = {}   # External tags: var_name => tag object
+        @external_tag_counter = 0
+        @has_external_filters = false  # Whether we need the filter helper
+      end
+
+      # Mark that we have external filters
+      def register_external_filter
+        @has_external_filters = true
+      end
+
+      # Check if external filters are used
+      def has_external_filters?
+        @has_external_filters
+      end
+
+      # Register an external tag that will be called at runtime
+      # @param tag [Liquid::Tag] The tag to register
+      # @return [String] The variable name for this tag
+      def register_external_tag(tag)
+        @external_tag_counter += 1
+        var_name = "__ext_tag_#{@external_tag_counter}__"
+        @external_tags[var_name] = tag
+        var_name
+      end
+
+      # Get all registered external tags
+      # @return [Hash] Map of variable names to tag objects
+      def external_tags
+        @external_tags
       end
 
       # Get the file system for loading partials
@@ -158,6 +187,7 @@ def extract_raw_markup(node)
 
       # Compile the template to a Ruby code string
       # @return [String] Ruby code that can be eval'd to create a render proc
+      # @return [Hash] If external tags are used, returns { code: String, external_tags: Hash }
       def compile
         code = CodeGenerator.new
 
@@ -169,8 +199,17 @@ def compile
           code.blank_line
         end
 
-        # Generate the lambda header
-        code.line "->(assigns = {}) do"
+        # First pass: compile the document body to discover partials and external tags
+        main_code = CodeGenerator.new
+        compile_node(@template.root, main_code)
+
+        # Determine lambda parameters based on external dependencies
+        params = ["assigns = {}"]
+        params << "__external_tags__ = {}" unless @external_tags.empty?
+        params << "__filter_handler__ = nil" if @has_external_filters
+
+        code.line "->(#{params.join(', ')}) do"
+
         code.indent do
           # Initialize the output buffer
           code.line '__output__ = +""'
@@ -182,9 +221,17 @@ def compile
             code.blank_line
           end
 
-          # First pass: compile the document body to discover partials
-          main_code = CodeGenerator.new
-          compile_node(@template.root, main_code)
+          # Add external tag runtime helper if needed
+          unless @external_tags.empty?
+            compile_external_tag_helper(code)
+            code.blank_line
+          end
+
+          # Add external filter helper if needed
+          if @has_external_filters
+            compile_filter_helper(code)
+            code.blank_line
+          end
 
           # Compile partial methods (before main body so they're available)
           compile_partials(code)
@@ -200,6 +247,41 @@ def compile
         code.to_s
       end
 
+      # Compile helper for calling external tags at runtime
+      def compile_external_tag_helper(code)
+        code.line "# Helper for calling external (unknown) tags at runtime"
+        code.line "__call_external_tag__ = ->(tag_var, tag_assigns) {"
+        code.indent do
+          code.line "tag = __external_tags__[tag_var]"
+          code.line "next '' unless tag"
+          code.line "# Create a context using the default environment (which has filters registered)"
+          code.line "ctx = Liquid::Context.new([tag_assigns], {}, {}, false, nil, {}, Liquid::Environment.default)"
+          code.line "output = +''"
+          code.line "# Use render_to_output_buffer to ensure block tags work correctly"
+          code.line "tag.render_to_output_buffer(ctx, output)"
+          code.line "output"
+        end
+        code.line "}"
+      end
+
+      # Compile helper for calling external filters at runtime
+      def compile_filter_helper(code)
+        code.line "# Helper for calling external (unknown) filters at runtime"
+        code.line "__call_filter__ = ->(name, input, args) {"
+        code.indent do
+          code.line "if __filter_handler__&.respond_to?(name)"
+          code.indent do
+            code.line "__filter_handler__.send(name, input, *args)"
+          end
+          code.line "else"
+          code.indent do
+            code.line "input # Return input unchanged if filter not found"
+          end
+          code.line "end"
+        end
+        code.line "}"
+      end
+
       # Compile all registered partials as inner methods
       def compile_partials(code)
         @partials.each do |name, method_name|
@@ -293,16 +375,27 @@ def compile_tag(tag, code)
         if compiler_class
           compiler_class.compile(tag, self, code)
         else
-          raise CompileError, "No compiler for tag: #{tag.class}"
+          # Unknown tag - delegate to the original tag's render method at runtime
+          compile_external_tag(tag, code)
         end
       end
 
+      def compile_external_tag(tag, code)
+        tag_var = register_external_tag(tag)
+        tag_name = tag.class.name.split('::').last
+        if debug?
+          code.line "# External tag: #{tag_name} (delegated to runtime)"
+          code.line "$stderr.puts '* WARN: Liquid external tag call - #{tag_name} (not compiled, delegated to runtime)' if $VERBOSE"
+        end
+        code.line "__output__ << __call_external_tag__.call(#{tag_var.inspect}, assigns)"
+      end
+
       def find_tag_compiler(tag)
         case tag
+        when Liquid::Unless  # Check Unless before If since Unless < If
+          Tags::UnlessCompiler
         when Liquid::If
           Tags::IfCompiler
-        when Liquid::Unless
-          Tags::UnlessCompiler
         when Liquid::Case
           Tags::CaseCompiler
         when Liquid::For

lib/liquid/compile/tags/capture_compiler.rb

@@ -15,11 +15,10 @@ def self.compile(tag, compiler, code)
           code.line "#{capture_var} = __output__"
           code.line "__output__ = +''"
 
-          # Compile the body
-          code.indent do
-            tag.nodelist.each do |body|
-              BlockBodyCompiler.compile(body, compiler, code)
-            end
+          # Compile the body - access the @body BlockBody directly
+          body = tag.instance_variable_get(:@body)
+          if body
+            BlockBodyCompiler.compile(body, compiler, code)
           end
 
           # Save captured content and restore output buffer

lib/liquid/compile/tags/include_compiler.rb

@@ -36,6 +36,10 @@ def self.compile_static_include(tag, template_name, compiler, code)
           partial_source = compiler.load_partial(template_name)
 
           if partial_source
+            if compiler.debug?
+              code.line "# Inlined partial '#{template_name}' at compile time"
+              code.line "$stderr.puts '* WARN: Liquid file system access - inlined partial \\\"#{template_name}\\\" at compile time' if $VERBOSE"
+            end
             # Generate a unique method name for this partial
             method_name = compiler.register_partial(template_name, partial_source)
             context_var_name = alias_name || template_name.split('/').last
@@ -89,6 +93,11 @@ def self.compile_dynamic_include(tag, compiler, code)
           attributes = tag.attributes
           alias_name = tag.instance_variable_get(:@alias_name)
 
+          if compiler.debug?
+            code.line "# Dynamic include (template name from variable)"
+            code.line "$stderr.puts '* WARN: Liquid runtime file system access - dynamic include (template name from variable)' if $VERBOSE"
+          end
+
           name_expr = ExpressionCompiler.compile(template_name_expr, compiler)
 
           # Build attributes hash

lib/liquid/compile/tags/render_compiler.rb

@@ -38,6 +38,10 @@ def self.compile_static_render(tag, template_name, compiler, code)
           partial_source = compiler.load_partial(template_name)
 
           if partial_source
+            if compiler.debug?
+              code.line "# Inlined partial '#{template_name}' at compile time"
+              code.line "$stderr.puts '* WARN: Liquid file system access - inlined partial \\\"#{template_name}\\\" at compile time' if $VERBOSE"
+            end
             # Generate a unique method name for this partial
             method_name = compiler.register_partial(template_name, partial_source)
             context_var_name = alias_name || template_name.split('/').last
@@ -146,6 +150,11 @@ def self.compile_dynamic_render(tag, compiler, code)
           alias_name = tag.alias_name
           is_for_loop = tag.for_loop?
 
+          if compiler.debug?
+            code.line "# Dynamic render (template name from variable)"
+            code.line "$stderr.puts '* WARN: Liquid runtime file system access - dynamic render (template name from variable)' if $VERBOSE"
+          end
+
           name_expr = ExpressionCompiler.compile(template_name_expr, compiler)
 
           # Build attributes hash