Merge pull request #30 from willtemperley/main

Initial work for dictionary arrays.

Author: willtemperley

Package.swift

@@ -20,7 +20,7 @@ import PackageDescription
 let package = Package(
   name: "Arrow",
   platforms: [
-    .macOS(.v26)
+    .macOS(.v26), .iOS(.v26), .watchOS(.v26), .tvOS(.v26), .visionOS(.v26),
   ],
   products: [
     .library(name: "Arrow", targets: ["Arrow"]),

Sources/Arrow/Array/Builder.swift

@@ -75,7 +75,7 @@ public class ArrayBuilderBoolean: AnyArrayBuilder {
 }
 
 /// A builder for Arrow arrays holding fixed-width types.
-public class ArrayBuilderFixedWidth<T: Numeric & BitwiseCopyable>:
+public class ArrayBuilderNumeric<T: Numeric & BitwiseCopyable>:
   AnyArrayBuilder
 {
 
@@ -234,7 +234,7 @@ typealias ArrayBuilderBinary = ArrayBuilderVariableLength<Data, Int32>
 /// A builder for Arrow arrays holding `Date`s with a resolution of one day.
 public struct ArrayBuilderDate32: AnyArrayBuilder {
   public typealias ArrayType = ArrowArrayDate32
-  let builder: ArrayBuilderFixedWidth<Date32> = .init()
+  let builder: ArrayBuilderNumeric<Date32> = .init()
 
   public init() {}
 
@@ -259,7 +259,7 @@ public struct ArrayBuilderDate32: AnyArrayBuilder {
 /// A builder for Arrow arrays holding `Date`s with a resolution of one day.
 public struct ArrayBuilderDate64: AnyArrayBuilder {
   public typealias ArrayType = ArrowArrayDate64
-  let builder: ArrayBuilderFixedWidth<Date64> = .init()
+  let builder: ArrayBuilderNumeric<Date64> = .init()
 
   public init() {}
 
@@ -282,13 +282,13 @@ public struct ArrayBuilderDate64: AnyArrayBuilder {
 }
 
 /// A builder for Arrow arrays holding Time32 values.
-public typealias ArrayBuilderTime32 = ArrayBuilderFixedWidth<Time32>
+public typealias ArrayBuilderTime32 = ArrayBuilderNumeric<Time32>
 
 /// A builder for Arrow arrays holding Time64 values.
-public typealias ArrayBuilderTime64 = ArrayBuilderFixedWidth<Time64>
+public typealias ArrayBuilderTime64 = ArrayBuilderNumeric<Time64>
 
 /// A builder for Arrow arrays holding Timestamp values.
-public typealias ArrayBuilderTimestamp = ArrayBuilderFixedWidth<Timestamp>
+public typealias ArrayBuilderTimestamp = ArrayBuilderNumeric<Timestamp>
 
 public class ArrayBuilderList<T: AnyArrayBuilder> {
 

Sources/Arrow/Array/DictionaryArray.swift

@@ -0,0 +1,68 @@
+// Copyright 2026 The Columnar Swift Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// A shared container for the dictionary values.
+public class DictionaryValues: @unchecked Sendable {
+  var currentArray: AnyArrowArrayProtocol
+
+  init(_ array: AnyArrowArrayProtocol) {
+    self.currentArray = array
+  }
+}
+
+///// An Arrow dictionary array.
+public struct ArrowDictionaryArray<
+  IndexType: FixedWidthInteger & BitwiseCopyable
+>: ArrowArrayProtocol {
+  public let offset: Int
+  public let length: Int
+  public var bufferSizes: [Int] { keys.bufferSizes }
+  public var buffers: [ArrowBufferProtocol] { keys.buffers }
+  public var nullCount: Int { keys.nullCount }
+
+  public let keys: ArrowArrayNumeric<IndexType>
+  public let values: DictionaryValues
+
+  public init(
+    offset: Int = 0,
+    length: Int,
+    keys: ArrowArrayNumeric<IndexType>,
+    values: AnyArrowArrayProtocol
+  ) {
+    self.offset = offset
+    self.length = length
+    self.keys = keys
+    self.values = DictionaryValues(values)
+  }
+
+  public subscript(index: Int) -> Any? {
+    precondition(index >= 0 && index < length, "Invalid index.")
+    let offsetIndex = self.offset + index
+    guard let key = keys[offsetIndex] else {
+      return nil
+    }
+    precondition(
+      Int(key) < values.currentArray.length, "Key out of bounds for dictionary")
+    return values.currentArray.any(at: Int(key))
+  }
+
+  public func slice(offset: Int, length: Int) -> Self {
+    .init(
+      offset: 0,
+      length: length,
+      keys: keys.slice(offset: self.offset + offset, length: length),
+      values: values.currentArray
+    )
+  }
+}

Sources/Arrow/ArrowField.swift

@@ -31,7 +31,6 @@ public struct ArrowField: Codable, Sendable {
   ///
   /// If true, the field *may* contain null values.
   public var isNullable: Bool
-  public let orderedDict: Bool
   /// A map of key-value pairs containing additional custom meta data.
   public var metadata: [String: String]
 }
@@ -60,7 +59,6 @@ extension ArrowField {
     self.name = name
     self.type = dataType
     self.isNullable = isNullable
-    self.orderedDict = false
     self.metadata = metadata
   }
 
@@ -76,21 +74,20 @@ extension ArrowField {
     )
   }
 
-  /// Create a new `ArrowField` suitable for `ArrowType::Dictionary`.
-  ///
-  public init(
-    dictWithName: String,
-    key: ArrowType,
-    value: ArrowType,
-    isNullable: Bool
-  ) {
-    precondition(
-      key.isDictionaryKeyType,
-      "\(key) is not a valid dictionary key"
-    )
-    let dataType: ArrowType = .dictionary(key, value)
-    self = Self(name: dictWithName, dataType: dataType, isNullable: isNullable)
-  }
+//  /// Create a new `ArrowField` suitable for `ArrowType::Dictionary`.
+//  public init(
+//    dictWithName: String,
+//    key: ArrowType,
+//    value: ArrowType,
+//    isNullable: Bool
+//  ) {
+//    precondition(
+//      key.isDictionaryKeyType,
+//      "\(key) is not a valid dictionary key"
+//    )
+//    let dataType: ArrowType = .dictionary(key, value)
+//    self = Self(name: dictWithName, dataType: dataType, isNullable: isNullable)
+//  }
 
   /// Create a new struct `ArrowField`.
   ///
@@ -218,7 +215,7 @@ extension ArrowField {
   @inlinable
   public var dictIsOrdered: Bool {
     switch self.type {
-    case .dictionary: return self.orderedDict
+    case .dictionary(_, let isOrdered, _, _): return isOrdered
     default: return false
     }
   }

Sources/Arrow/ArrowType.swift

@@ -293,7 +293,7 @@ public indirect enum ArrowType: Codable, Sendable, Equatable {
   ///
   /// This type mostly used to represent low cardinality string
   /// arrays or a limited set of primitive types as integers.
-  case dictionary(ArrowType, ArrowType)
+  case dictionary(id: Int64, isOrdered: Bool, key: ArrowType, value: ArrowType)
   /// Exact 32-bit width decimal value with precision and scale
   ///
   /// * precision is the total number of digits
@@ -496,8 +496,8 @@ extension ArrowType: CustomStringConvertible {
       return "LargeListView(\(elementType))"
     case .union(let mode, let fields):
       return "Union(\(mode), \(fields) fields)"
-    case .dictionary(let keyType, let valueType):
-      return "Dictionary(\(keyType), \(valueType))"
+    case .dictionary(let id, let isOrdered, let keyType, let valueType):
+      return "Dictionary(\(id), \(isOrdered), \(keyType), \(valueType))"
     case .decimal32(let precision, let scale):
       return "Decimal32(\(precision), \(scale))"
     case .decimal64(let precision, let scale):
@@ -627,7 +627,7 @@ extension ArrowType {
   @inlinable
   public var isNested: Bool {
     switch self {
-    case .dictionary(_, let v):
+    case .dictionary(_, _, _, let v):
       return v.isNested
     case .runEndEncoded(_, let v):
       return v.type.isNested
@@ -699,7 +699,8 @@ extension ArrowType {
         && aField.type.equalsDataType(bField.type) && aSorted == bSorted
 
     // Dictionary
-    case (.dictionary(let aKey, let aValue), .dictionary(let bKey, let bValue)):
+    case (.dictionary(_, _, let aKey, let aValue), .dictionary(_, _, let bKey, let bValue)):
+      // Ignoring dictionary id here.
       return aKey.equalsDataType(bKey) && aValue.equalsDataType(bValue)
 
     // RunEndEncoded
@@ -855,7 +856,7 @@ extension ArrowType {
       }
 
     // Dictionary
-    case (.dictionary(let k1, let v1), .dictionary(let k2, let v2)):
+    case (.dictionary(_, _, let k1, let v1), .dictionary(_, _, let k2, let v2)):
       return k1.contains(k2) && v1.contains(v2)
 
     // Base case: equality

Sources/ArrowIPC/ArrowReader.swift

@@ -90,6 +90,51 @@ public struct ArrowReader {
     let arrowSchema = try Self.loadSchema(schema: schema)
     var recordBatches: [RecordBatch] = []
 
+    for block in footer.dictionaries {
+      let (message, bodyOffset) = try data.withParserSpan { input in
+        try input.seek(toAbsoluteOffset: block.offset)
+        let marker = try UInt32(parsingLittleEndian: &input)
+        if marker != continuationMarker {
+          throw ArrowError(.invalid("Missing continuation marker."))
+        }
+        let messageLength = try UInt32(parsingLittleEndian: &input)
+        let data = try [UInt8](parsing: &input, byteCount: Int(messageLength))
+        var mbb = ByteBuffer(data: Data(data))
+        let message: FMessage = getRoot(byteBuffer: &mbb)
+        let offset = Int64(input.startPosition)
+        return (message, offset)
+      }
+
+      guard message.headerType == .dictionarybatch else {
+        throw ArrowError(.invalid("Expected DictionaryBatch message."))
+      }
+
+      guard let dictMessage = message.header(type: FDictionaryBatch.self) else {
+        throw ArrowError(.invalid("Expected DictionaryBatch as message header"))
+      }
+
+      // 1. Get the Dictionary ID and 'isDelta' flag
+      let dictId = dictMessage.id
+      let isDelta = dictMessage.isDelta
+
+      // 2. The dictionary data is actually just a RecordBatch with ONE column
+      // The schema for this internal batch is defined by the dictionary type
+      // found in the global Schema for this specific ID.
+      guard let rbMessage = dictMessage.data else {
+        throw ArrowError(.invalid("DictionaryBatch has no data"))
+      }
+
+      let dictBatch = try Self.loadRecordBatch(
+        data: self.data,
+        arrowSchema: arrowSchema,
+        rbMessage: rbMessage,
+        offset: bodyOffset
+      )
+
+      // 4. Update the "Box" in your provider
+      //        try dictionaryProvider.update(id: dictId, array: dictionaryArray, isDelta: isDelta)
+    }
+
     // MARK: Record batch parsing
     for block in footer.recordBatches {
 

Sources/ArrowIPC/FlatBuffersTypeAliases.swift

@@ -13,16 +13,22 @@
 // limitations under the License.
 
 typealias FFooter = org_apache_arrow_flatbuf_Footer
+
+typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader
 typealias FMessage = org_apache_arrow_flatbuf_Message
+
 typealias FBlock = org_apache_arrow_flatbuf_Block
 typealias FField = org_apache_arrow_flatbuf_Field
 typealias FSchema = org_apache_arrow_flatbuf_Schema
 typealias FBuffer = org_apache_arrow_flatbuf_Buffer
 typealias FFieldNode = org_apache_arrow_flatbuf_FieldNode
-typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch
-typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader
 typealias FKeyValue = org_apache_arrow_flatbuf_KeyValue
 
+// MARK: Record batches.
+typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch
+typealias FDictionaryBatch = org_apache_arrow_flatbuf_DictionaryBatch
+typealias FDictionaryEncoding = org_apache_arrow_flatbuf_DictionaryEncoding
+
 // MARK: Top level type.
 typealias FType = org_apache_arrow_flatbuf_Type_
 

Tests/ArrowIPCTests/ArrowTestingGold.swift

@@ -82,10 +82,10 @@ struct ArrowTestingGold {
     "generated_union",
   ]
 
-  @Test
+//  @Test
   func readFile() throws {
 
-    let name = "generated_nested_large_offsets"
+    let name = "generated_dictionary"
     let (testFile, testCase) = try loadTestCase(
       name: name, fileExtension: "arrow_file")
     let arrowReader = try ArrowReader(url: testFile)

Tests/ArrowTests/ArrayNullBufferTests.swift

@@ -21,7 +21,7 @@ struct ArrayNullBufferTests {
 
   @Test func allValidValues() throws {
     // Should be able to omit null buffer entirely
-    let arrayBuilder: ArrayBuilderFixedWidth<Int64> = .init()
+    let arrayBuilder: ArrayBuilderNumeric<Int64> = .init()
     for i in 0..<1000 {
       arrayBuilder.append(Int64(i))  // No nulls
     }
@@ -35,7 +35,7 @@ struct ArrayNullBufferTests {
   }
 
   @Test func allNullValues() throws {
-    let arrayBuilder: ArrayBuilderFixedWidth<Int64> = .init()
+    let arrayBuilder: ArrayBuilderNumeric<Int64> = .init()
     for _ in 0..<1000 {
       arrayBuilder.appendNull()
     }

Tests/ArrowTests/BasicArrayTests.swift

@@ -38,7 +38,7 @@ struct BasicArrayTests {
   }
 
   @Test func uint8Array() throws {
-    let arrayBuilder: ArrayBuilderFixedWidth<UInt8> = .init()
+    let arrayBuilder: ArrayBuilderNumeric<UInt8> = .init()
     for index: UInt8 in 0..<100 {
       arrayBuilder.append(index)
     }
@@ -105,7 +105,7 @@ struct BasicArrayTests {
   }
 
   @Test func doubleArray() throws {
-    let builder: ArrayBuilderFixedWidth<Double> = .init()
+    let builder: ArrayBuilderNumeric<Double> = .init()
     builder.append(14)
     builder.appendNull()
     builder.append(40.4)