GH-48347: [Ruby] Add support for reading list array (#48351)

### Rationale for this change

This is the first nested type.

### What changes are included in this PR?

* Add `ArrowFormat::ListType`
* Add `ArrowFormat::ListArray`

### Are these changes tested?

Yes.

### Are there any user-facing changes?

Yes.
* GitHub Issue: #48347

Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>

Author: kou

ruby/red-arrow-format/lib/arrow-format/array.rb

@@ -123,4 +123,23 @@ def encoding
       Encoding::UTF_8
     end
   end
+
+  class ListArray < Array
+    def initialize(type, size, validity_buffer, offsets_buffer, child)
+      super(type, size, validity_buffer)
+      @offsets_buffer = offsets_buffer
+      @child = child
+    end
+
+    def to_a
+      child_values = @child.to_a
+      values = @offsets_buffer.
+        each(:s32, 0, @size + 1). # TODO: big endian support
+        each_cons(2).
+        collect do |(_, offset), (_, next_offset)|
+        child_values[offset...next_offset]
+      end
+      apply_validity(values)
+    end
+  end
 end

ruby/red-arrow-format/lib/arrow-format/file-reader.rb

@@ -22,14 +22,15 @@
 require_relative "schema"
 require_relative "type"
 
+require_relative "org/apache/arrow/flatbuf/binary"
 require_relative "org/apache/arrow/flatbuf/bool"
 require_relative "org/apache/arrow/flatbuf/footer"
-require_relative "org/apache/arrow/flatbuf/message"
-require_relative "org/apache/arrow/flatbuf/binary"
 require_relative "org/apache/arrow/flatbuf/int"
+require_relative "org/apache/arrow/flatbuf/list"
+require_relative "org/apache/arrow/flatbuf/message"
 require_relative "org/apache/arrow/flatbuf/null"
-require_relative "org/apache/arrow/flatbuf/utf8"
 require_relative "org/apache/arrow/flatbuf/schema"
+require_relative "org/apache/arrow/flatbuf/utf8"
 
 module ArrowFormat
   class FileReader
@@ -90,9 +91,10 @@ def each
         when Org::Apache::Arrow::Flatbuf::RecordBatch
           n_rows = header.length
           columns = []
+          nodes = header.nodes
           buffers = header.buffers
           schema.fields.each do |field|
-            columns << read_column(field, n_rows, buffers, body)
+            columns << read_column(field, nodes, buffers, body)
           end
           yield(RecordBatch.new(schema, n_rows, columns))
         end
@@ -129,35 +131,44 @@ def read_footer
       Org::Apache::Arrow::Flatbuf::Footer.new(footer_data)
     end
 
-    def read_schema(fb_schema)
-      fields = fb_schema.fields.collect do |fb_field|
-        fb_type = fb_field.type
-        case fb_type
-        when Org::Apache::Arrow::Flatbuf::Null
-          type = NullType.singleton
-        when Org::Apache::Arrow::Flatbuf::Bool
-          type = BooleanType.singleton
-        when Org::Apache::Arrow::Flatbuf::Int
-          case fb_type.bit_width
-          when 8
-            if fb_type.signed?
-              type = Int8Type.singleton
-            else
-              type = UInt8Type.singleton
-            end
+    def read_field(fb_field)
+      fb_type = fb_field.type
+      case fb_type
+      when Org::Apache::Arrow::Flatbuf::Null
+        type = NullType.singleton
+      when Org::Apache::Arrow::Flatbuf::Bool
+        type = BooleanType.singleton
+      when Org::Apache::Arrow::Flatbuf::Int
+        case fb_type.bit_width
+        when 8
+          if fb_type.signed?
+            type = Int8Type.singleton
+          else
+            type = UInt8Type.singleton
           end
-        when Org::Apache::Arrow::Flatbuf::Binary
-          type = BinaryType.singleton
-        when Org::Apache::Arrow::Flatbuf::Utf8
-          type = UTF8Type.singleton
         end
-        Field.new(fb_field.name, type)
+      when Org::Apache::Arrow::Flatbuf::List
+        type = ListType.new(read_field(fb_field.children[0]))
+      when Org::Apache::Arrow::Flatbuf::Binary
+        type = BinaryType.singleton
+      when Org::Apache::Arrow::Flatbuf::Utf8
+        type = UTF8Type.singleton
+      end
+      Field.new(fb_field.name, type)
+    end
+
+    def read_schema(fb_schema)
+      fields = fb_schema.fields.collect do |fb_field|
+        read_field(fb_field)
       end
       Schema.new(fields)
     end
 
-    def read_column(field, n_rows, buffers, body)
-      return field.type.build_array(n_rows) if field.type.is_a?(NullType)
+    def read_column(field, nodes, buffers, body)
+      node = nodes.shift
+      length = node.length
+
+      return field.type.build_array(length) if field.type.is_a?(NullType)
 
       validity_buffer = buffers.shift
       if validity_buffer.length.zero?
@@ -172,14 +183,19 @@ def read_column(field, n_rows, buffers, body)
            UInt8Type
         values_buffer = buffers.shift
         values = body.slice(values_buffer.offset, values_buffer.length)
-        field.type.build_array(n_rows, validity, values)
+        field.type.build_array(length, validity, values)
+      when ListType
+        offsets_buffer = buffers.shift
+        offsets = body.slice(offsets_buffer.offset, offsets_buffer.length)
+        child = read_column(field.type.child, nodes, buffers, body)
+        field.type.build_array(length, validity, offsets, child)
       when BinaryType,
            UTF8Type
         offsets_buffer = buffers.shift
         values_buffer = buffers.shift
         offsets = body.slice(offsets_buffer.offset, offsets_buffer.length)
         values = body.slice(values_buffer.offset, values_buffer.length)
-        field.type.build_array(n_rows, validity, offsets, values)
+        field.type.build_array(length, validity, offsets, values)
       end
     end
   end

ruby/red-arrow-format/lib/arrow-format/type.rb

@@ -103,7 +103,6 @@ def singleton
       end
     end
 
-    attr_reader :name
     def initialize
       super("Binary")
     end
@@ -129,4 +128,16 @@ def build_array(size, validity_buffer, offsets_buffer, values_buffer)
       UTF8Array.new(self, size, validity_buffer, offsets_buffer, values_buffer)
     end
   end
+
+  class ListType < Type
+    attr_reader :child
+    def initialize(child)
+      super("List")
+      @child = child
+    end
+
+    def build_array(size, validity_buffer, offsets_buffer, child)
+      ListArray.new(self, size, validity_buffer, offsets_buffer, child)
+    end
+  end
 end

ruby/red-arrow-format/test/test-file-reader.rb

@@ -105,4 +105,16 @@ def test_read
                    read)
     end
   end
+
+  sub_test_case("List") do
+    def build_array
+      data_type = Arrow::ListDataType.new(name: "count", type: :int8)
+      Arrow::ListArray.new(data_type, [[-128, 127], nil, [-1, 0, 1]])
+    end
+
+    def test_read
+      assert_equal([{"value" => [[-128, 127], nil, [-1, 0, 1]]}],
+                   read)
+    end
+  end
 end