Merge pull request #22 from willtemperley/main

Use concrete buffer types to build arrays. Contain generics in typealiases.

Author: willtemperley

Sources/Arrow/Array/Array.swift

@@ -70,14 +70,18 @@ public struct ArrowArrayBoolean: ArrowArrayProtocol {
 }
 
 /// An Arrow array of fixed-width types.
-struct ArrowArrayFixed<T>: ArrowArrayProtocol where T: Numeric {
-  typealias ItemType = T
+struct ArrowArrayFixed<Element, ValueBuffer>: ArrowArrayProtocol
+where
+  Element: Numeric,
+  ValueBuffer: FixedWidthBufferProtocol<Element>
+{
+  typealias ItemType = Element
   let offset: Int
   let length: Int
   let nullBuffer: NullBuffer
-  let valueBuffer: FixedWidthBuffer<T>
+  let valueBuffer: ValueBuffer
 
-  subscript(index: Int) -> T? {
+  subscript(index: Int) -> Element? {
     precondition(index >= 0 && index < length, "Invalid index.")
     let offsetIndex = self.offset + index
     if !self.nullBuffer.isSet(offsetIndex) {
@@ -86,7 +90,7 @@ struct ArrowArrayFixed<T>: ArrowArrayProtocol where T: Numeric {
     return valueBuffer[offsetIndex]
   }
 
-  func slice(offset: Int, length: Int) -> ArrowArrayFixed<T> {
+  func slice(offset: Int, length: Int) -> Self {
     .init(
       offset: offset,
       length: length,
@@ -97,22 +101,26 @@ struct ArrowArrayFixed<T>: ArrowArrayProtocol where T: Numeric {
 }
 
 /// An Arrow array of variable-length types.
-public struct ArrowArrayVariable<T>: ArrowArrayProtocol
-where T: VariableLength {
-  public typealias ItemType = T
+public struct ArrowArrayVariable<Element, OffsetsBuffer, ValueBuffer>:
+  ArrowArrayProtocol
+where
+  Element: VariableLength,
+  OffsetsBuffer: FixedWidthBufferProtocol<Int32>,
+  ValueBuffer: VariableLengthBufferProtocol<Element>
+{
+  public typealias ItemType = Element
   public let offset: Int
   public let length: Int
   let nullBuffer: NullBuffer
-  let offsetsBuffer: any FixedWidthBufferProtocol<Int32>
-  let valueBuffer: any VariableLengthBufferProtocol<T>
+  let offsetsBuffer: OffsetsBuffer
+  let valueBuffer: ValueBuffer
 
   public init(
     offset: Int,
     length: Int,
     nullBuffer: NullBuffer,
-    offsetsBuffer:
-      any FixedWidthBufferProtocol<Int32>,
-    valueBuffer: any VariableLengthBufferProtocol<T>
+    offsetsBuffer: OffsetsBuffer,
+    valueBuffer: ValueBuffer
   ) {
     self.offset = offset
     self.length = length
@@ -121,7 +129,7 @@ where T: VariableLength {
     self.valueBuffer = valueBuffer
   }
 
-  public subscript(index: Int) -> T? {
+  public subscript(index: Int) -> Element? {
 
     let offsetIndex = self.offset + index
 
@@ -147,14 +155,14 @@ where T: VariableLength {
   }
 }
 
-public typealias ArrowArrayUtf8 = ArrowArrayVariable<String>
-public typealias ArrowArrayBinary = ArrowArrayVariable<Data>
-
 /// An Arrow array of `Date`s with a resolution of 1 day.
-struct ArrowArrayDate32: ArrowArrayProtocol {
+struct ArrowArrayDate32<ValueBuffer>: ArrowArrayProtocol
+where
+  ValueBuffer: FixedWidthBufferProtocol<Int32>
+{
   typealias ItemType = Date
 
-  let array: ArrowArrayFixed<Date32>
+  let array: ArrowArrayFixed<Date32, ValueBuffer>
 
   var offset: Int {
     array.offset
@@ -182,10 +190,13 @@ struct ArrowArrayDate32: ArrowArrayProtocol {
 }
 
 /// An Arrow array of `Date`s with a resolution of 1 second.
-struct ArrowArrayDate64: ArrowArrayProtocol {
+struct ArrowArrayDate64<ValueBuffer>: ArrowArrayProtocol
+where
+  ValueBuffer: FixedWidthBufferProtocol<Int64>
+{
   typealias ItemType = Date
 
-  let array: ArrowArrayFixed<Date64>
+  let array: ArrowArrayFixed<Date64, ValueBuffer>
 
   var offset: Int {
     array.offset
@@ -212,17 +223,20 @@ struct ArrowArrayDate64: ArrowArrayProtocol {
   }
 }
 
-struct ArrowListArray<T>: ArrowArrayProtocol where T: ArrowArrayProtocol {
-
-  typealias ItemType = T
+/// An Arrow list array which may be nested arbitrarily.
+struct ArrowListArray<Element>: ArrowArrayProtocol
+where
+  Element: ArrowArrayProtocol
+{
+  typealias ItemType = Element
 
   let offset: Int
   let length: Int
   let nullBuffer: NullBuffer
   let offsetsBuffer: FixedWidthBuffer<Int32>
-  let values: T
+  let values: Element
 
-  subscript(index: Int) -> T? {
+  subscript(index: Int) -> Element? {
     precondition(index >= 0 && index < length, "Invalid index.")
     let offsetIndex = self.offset + index
     if !self.nullBuffer.isSet(offsetIndex) {

Sources/Arrow/Array/Builder.swift

@@ -76,7 +76,7 @@ class ArrayBuilderBoolean: AnyArrayBuilder {
 /// A builder for Arrow arrays holding fixed-width types.
 class ArrayBuilderFixedWidth<T: Numeric>: AnyArrayBuilder {
 
-  typealias ArrayType = ArrowArrayFixed<T>
+  typealias ArrayType = ArrowArrayFixed<T, FixedWidthBuffer<T>>
 
   var length: Int
   let nullBuilder: NullBufferBuilder
@@ -114,23 +114,25 @@ class ArrayBuilderFixedWidth<T: Numeric>: AnyArrayBuilder {
 }
 
 /// A builder for Arrow arrays holding variable length types.
-class ArrayBuilderVariable<T: VariableLength>: AnyArrayBuilder {
-  typealias ArrayType = ArrowArrayVariable<T>
+class ArrayBuilderVariable<Element: VariableLength>: AnyArrayBuilder {
+  typealias ArrayType = ArrowArrayVariable<
+    Element, FixedWidthBuffer<Int32>, VariableLengthTypeBuffer<Element>
+  >
 
   var length: Int
   let nullBuilder: NullBufferBuilder
   let offsetsBuilder: FixedWidthBufferBuilder<Int32>
-  let valueBuilder: VariableLengthTypeBufferBuilder<T>
+  let valueBuilder: VariableLengthTypeBufferBuilder<Element>
 
   init() {
     self.length = 0
     self.nullBuilder = NullBufferBuilder()
     self.offsetsBuilder = FixedWidthBufferBuilder<Int32>()
-    self.valueBuilder = VariableLengthTypeBufferBuilder<T>()
+    self.valueBuilder = VariableLengthTypeBufferBuilder<Element>()
     self.offsetsBuilder.append(Int32.zero)
   }
 
-  public func append(_ value: T) {
+  public func append(_ value: Element) {
     length += 1
     nullBuilder.appendValid(true)
     let data = value.data
@@ -176,7 +178,7 @@ typealias ArrayBuilderBinary = ArrayBuilderVariable<Data>
 
 /// A builder for Arrow arrays holding `Date`s with a resolution of one day.
 struct ArrayBuilderDate32: AnyArrayBuilder {
-  typealias ArrayType = ArrowArrayDate32
+  typealias ArrayType = ArrowArrayDate32<FixedWidthBuffer<Date32>>
 
   let builder: ArrayBuilderFixedWidth<Date32> = .init()
 
@@ -200,7 +202,7 @@ struct ArrayBuilderDate32: AnyArrayBuilder {
 
 /// A builder for Arrow arrays holding `Date`s with a resolution of one day.
 struct ArrayBuilderDate64: AnyArrayBuilder {
-  typealias ArrayType = ArrowArrayDate64
+  typealias ArrayType = ArrowArrayDate64<FixedWidthBuffer<Date64>>
 
   let builder: ArrayBuilderFixedWidth<Date64> = .init()
 

Sources/Arrow/Buffer/NullBuffer.swift

@@ -48,10 +48,10 @@ public struct AllNullBuffer: NullBuffer {
 
 /// A  bit-packed buffer used to represent nulls and booleans in Arrow arrays.
 final class BitPackedNullBuffer: NullBuffer {
-  var length: Int
-  var capacity: Int
+  let length: Int
+  let capacity: Int
   let ownsMemory: Bool
-  var buffer: UnsafePointer<UInt8>
+  let buffer: UnsafePointer<UInt8>
 
   init(
     length: Int,

Sources/ArrowIPC/Array+IPC.swift

@@ -0,0 +1,82 @@
+// Copyright 2025 The Columnar Swift Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Arrow
+import Foundation
+
+typealias ArrowArrayUtf8 = ArrowArrayVariable<
+  String,
+  FixedWidthBufferIPC<Int32>,
+  VariableLengthBufferIPC<String>
+>
+
+extension ArrowArrayUtf8 {
+
+  /// Build a `Data` backed Arrow utf8 array.
+  /// - Parameters:
+  ///   - length: The array length.
+  ///   - nullBuffer: The null buffer.
+  ///   - offsetsBuffer: A view over file-backed data.
+  ///   - valueBuffer: A view over file-backed data.
+  /// - Returns: A file-backed Arrow utf8 array.
+  static func utf8(
+    length: Int,
+    nullBuffer: NullBuffer,
+    offsetsBuffer: FileDataBuffer,
+    valueBuffer: FileDataBuffer
+  ) -> Self {
+    let offsetsBufferTyped = FixedWidthBufferIPC<Int32>(buffer: offsetsBuffer)
+    let valueBufferTyped = VariableLengthBufferIPC<String>(buffer: valueBuffer)
+    return Self(
+      offset: 0,
+      length: length,
+      nullBuffer: nullBuffer,
+      offsetsBuffer: offsetsBufferTyped,
+      valueBuffer: valueBufferTyped
+    )
+  }
+}
+
+typealias ArrowArrayBinary = ArrowArrayVariable<
+  Data,
+  FixedWidthBufferIPC<Int32>,
+  VariableLengthBufferIPC<Data>
+>
+
+extension ArrowArrayBinary {
+
+  /// Build a `Data` backed Arrow binary array.
+  /// - Parameters:
+  ///   - length: The array length.
+  ///   - nullBuffer: The null buffer.
+  ///   - offsetsBuffer: A view over file-backed data.
+  ///   - valueBuffer: A view over file-backed data.
+  /// - Returns: A file-backed Arrow utf8 array.
+  static func binary(
+    length: Int,
+    nullBuffer: NullBuffer,
+    offsetsBuffer: FileDataBuffer,
+    valueBuffer: FileDataBuffer
+  ) -> Self {
+    let offsetsBufferTyped = FixedWidthBufferIPC<Int32>(buffer: offsetsBuffer)
+    let valueBufferTyped = VariableLengthBufferIPC<Data>(buffer: valueBuffer)
+    return Self(
+      offset: 0,
+      length: length,
+      nullBuffer: nullBuffer,
+      offsetsBuffer: offsetsBufferTyped,
+      valueBuffer: valueBufferTyped
+    )
+  }
+}