GH-48360: [Ruby] Add support for reading large binary array

Author: kou

ruby/red-arrow-format/lib/arrow-format/array.rb

@@ -100,7 +100,7 @@ def initialize(type, size, validity_buffer, offsets_buffer, values_buffer)
 
     def to_a
       values = @offsets_buffer.
-        each(:s32, 0, @size + 1). # TODO: big endian support
+        each(buffer_type, 0, @size + 1).
         each_cons(2).
         collect do |(_, offset), (_, next_offset)|
         length = next_offset - offset
@@ -112,13 +112,32 @@ def to_a
 
   class BinaryArray < VariableSizeBinaryLayoutArray
     private
+    def buffer_type
+      :s32 # TODO: big endian support
+    end
+
+    def encoding
+      Encoding::ASCII_8BIT
+    end
+  end
+
+  class LargeBinaryArray < VariableSizeBinaryLayoutArray
+    private
+    def buffer_type
+      :s64 # TODO: big endian support
+    end
+
     def encoding
       Encoding::ASCII_8BIT
     end
   end
 
   class UTF8Array < VariableSizeBinaryLayoutArray
     private
+    def buffer_type
+      :s32 # TODO: big endian support
+    end
+
     def encoding
       Encoding::UTF_8
     end

ruby/red-arrow-format/lib/arrow-format/file-reader.rb

@@ -26,6 +26,7 @@
 require_relative "org/apache/arrow/flatbuf/bool"
 require_relative "org/apache/arrow/flatbuf/footer"
 require_relative "org/apache/arrow/flatbuf/int"
+require_relative "org/apache/arrow/flatbuf/large_binary"
 require_relative "org/apache/arrow/flatbuf/list"
 require_relative "org/apache/arrow/flatbuf/message"
 require_relative "org/apache/arrow/flatbuf/null"
@@ -151,6 +152,8 @@ def read_field(fb_field)
         type = ListType.new(read_field(fb_field.children[0]))
       when Org::Apache::Arrow::Flatbuf::Binary
         type = BinaryType.singleton
+      when Org::Apache::Arrow::Flatbuf::LargeBinary
+        type = LargeBinaryType.singleton
       when Org::Apache::Arrow::Flatbuf::Utf8
         type = UTF8Type.singleton
       end
@@ -189,8 +192,7 @@ def read_column(field, nodes, buffers, body)
         offsets = body.slice(offsets_buffer.offset, offsets_buffer.length)
         child = read_column(field.type.child, nodes, buffers, body)
         field.type.build_array(length, validity, offsets, child)
-      when BinaryType,
-           UTF8Type
+      when VariableSizeBinaryType
         offsets_buffer = buffers.shift
         values_buffer = buffers.shift
         offsets = body.slice(offsets_buffer.offset, offsets_buffer.length)

ruby/red-arrow-format/lib/arrow-format/type.rb

@@ -96,7 +96,10 @@ def build_array(size, validity_buffer, values_buffer)
     end
   end
 
-  class BinaryType < Type
+  class VariableSizeBinaryType < Type
+  end
+
+  class BinaryType < VariableSizeBinaryType
     class << self
       def singleton
         @singleton ||= new
@@ -112,7 +115,27 @@ def build_array(size, validity_buffer, offsets_buffer, values_buffer)
     end
   end
 
-  class UTF8Type < Type
+  class LargeBinaryType < VariableSizeBinaryType
+    class << self
+      def singleton
+        @singleton ||= new
+      end
+    end
+
+    def initialize
+      super("LargeBinary")
+    end
+
+    def build_array(size, validity_buffer, offsets_buffer, values_buffer)
+      LargeBinaryArray.new(self,
+                           size,
+                           validity_buffer,
+                           offsets_buffer,
+                           values_buffer)
+    end
+  end
+
+  class UTF8Type < VariableSizeBinaryType
     class << self
       def singleton
         @singleton ||= new

ruby/red-arrow-format/test/test-file-reader.rb

@@ -95,6 +95,17 @@ def test_read
     end
   end
 
+  sub_test_case("LargeBinary") do
+    def build_array
+      Arrow::LargeBinaryArray.new(["Hello".b, nil, "World".b])
+    end
+
+    def test_read
+      assert_equal([{"value" => ["Hello".b, nil, "World".b]}],
+                   read)
+    end
+  end
+
   sub_test_case("UTF8") do
     def build_array
       Arrow::StringArray.new(["Hello", nil, "World"])