Data sources menu fixed..

Author: willtemperley

Package.swift

@@ -20,7 +20,7 @@ import PackageDescription
 let package = Package(
   name: "Arrow",
   platforms: [
-    .macOS(.v26)
+    .macOS(.v26), .iOS(.v26), .watchOS(.v26), .tvOS(.v26), .visionOS(.v26),
   ],
   products: [
     .library(name: "Arrow", targets: ["Arrow"]),

Sources/Arrow/ArrowField.swift

@@ -31,7 +31,6 @@ public struct ArrowField: Codable, Sendable {
   ///
   /// If true, the field *may* contain null values.
   public var isNullable: Bool
-  public let orderedDict: Bool
   /// A map of key-value pairs containing additional custom meta data.
   public var metadata: [String: String]
 }
@@ -60,7 +59,6 @@ extension ArrowField {
     self.name = name
     self.type = dataType
     self.isNullable = isNullable
-    self.orderedDict = false
     self.metadata = metadata
   }
 
@@ -76,21 +74,20 @@ extension ArrowField {
     )
   }
 
-  /// Create a new `ArrowField` suitable for `ArrowType::Dictionary`.
-  ///
-  public init(
-    dictWithName: String,
-    key: ArrowType,
-    value: ArrowType,
-    isNullable: Bool
-  ) {
-    precondition(
-      key.isDictionaryKeyType,
-      "\(key) is not a valid dictionary key"
-    )
-    let dataType: ArrowType = .dictionary(key, value)
-    self = Self(name: dictWithName, dataType: dataType, isNullable: isNullable)
-  }
+//  /// Create a new `ArrowField` suitable for `ArrowType::Dictionary`.
+//  public init(
+//    dictWithName: String,
+//    key: ArrowType,
+//    value: ArrowType,
+//    isNullable: Bool
+//  ) {
+//    precondition(
+//      key.isDictionaryKeyType,
+//      "\(key) is not a valid dictionary key"
+//    )
+//    let dataType: ArrowType = .dictionary(key, value)
+//    self = Self(name: dictWithName, dataType: dataType, isNullable: isNullable)
+//  }
 
   /// Create a new struct `ArrowField`.
   ///
@@ -218,7 +215,7 @@ extension ArrowField {
   @inlinable
   public var dictIsOrdered: Bool {
     switch self.type {
-    case .dictionary: return self.orderedDict
+    case .dictionary(_, let isOrdered, _, _): return isOrdered
     default: return false
     }
   }

Sources/Arrow/ArrowType.swift

@@ -293,7 +293,7 @@ public indirect enum ArrowType: Codable, Sendable, Equatable {
   ///
   /// This type mostly used to represent low cardinality string
   /// arrays or a limited set of primitive types as integers.
-  case dictionary(ArrowType, ArrowType)
+  case dictionary(id: Int64, isOrdered: Bool, key: ArrowType, value: ArrowType)
   /// Exact 32-bit width decimal value with precision and scale
   ///
   /// * precision is the total number of digits
@@ -496,8 +496,8 @@ extension ArrowType: CustomStringConvertible {
       return "LargeListView(\(elementType))"
     case .union(let mode, let fields):
       return "Union(\(mode), \(fields) fields)"
-    case .dictionary(let keyType, let valueType):
-      return "Dictionary(\(keyType), \(valueType))"
+    case .dictionary(let id, let isOrdered, let keyType, let valueType):
+      return "Dictionary(\(id), \(isOrdered), \(keyType), \(valueType))"
     case .decimal32(let precision, let scale):
       return "Decimal32(\(precision), \(scale))"
     case .decimal64(let precision, let scale):
@@ -627,7 +627,7 @@ extension ArrowType {
   @inlinable
   public var isNested: Bool {
     switch self {
-    case .dictionary(_, let v):
+    case .dictionary(_, _, _, let v):
       return v.isNested
     case .runEndEncoded(_, let v):
       return v.type.isNested
@@ -699,7 +699,8 @@ extension ArrowType {
         && aField.type.equalsDataType(bField.type) && aSorted == bSorted
 
     // Dictionary
-    case (.dictionary(let aKey, let aValue), .dictionary(let bKey, let bValue)):
+    case (.dictionary(_, _, let aKey, let aValue), .dictionary(_, _, let bKey, let bValue)):
+      // Ignoring dictionary id here.
       return aKey.equalsDataType(bKey) && aValue.equalsDataType(bValue)
 
     // RunEndEncoded
@@ -855,7 +856,7 @@ extension ArrowType {
       }
 
     // Dictionary
-    case (.dictionary(let k1, let v1), .dictionary(let k2, let v2)):
+    case (.dictionary(_, _, let k1, let v1), .dictionary(_, _, let k2, let v2)):
       return k1.contains(k2) && v1.contains(v2)
 
     // Base case: equality

Sources/ArrowIPC/ArrowReader.swift

@@ -90,6 +90,51 @@ public struct ArrowReader {
     let arrowSchema = try Self.loadSchema(schema: schema)
     var recordBatches: [RecordBatch] = []
 
+    for block in footer.dictionaries {
+      let (message, bodyOffset) = try data.withParserSpan { input in
+        try input.seek(toAbsoluteOffset: block.offset)
+        let marker = try UInt32(parsingLittleEndian: &input)
+        if marker != continuationMarker {
+          throw ArrowError(.invalid("Missing continuation marker."))
+        }
+        let messageLength = try UInt32(parsingLittleEndian: &input)
+        let data = try [UInt8](parsing: &input, byteCount: Int(messageLength))
+        var mbb = ByteBuffer(data: Data(data))
+        let message: FMessage = getRoot(byteBuffer: &mbb)
+        let offset = Int64(input.startPosition)
+        return (message, offset)
+      }
+
+      guard message.headerType == .dictionarybatch else {
+        throw ArrowError(.invalid("Expected DictionaryBatch message."))
+      }
+
+      guard let dictMessage = message.header(type: FDictionaryBatch.self) else {
+        throw ArrowError(.invalid("Expected DictionaryBatch as message header"))
+      }
+
+      // 1. Get the Dictionary ID and 'isDelta' flag
+      let dictId = dictMessage.id
+      let isDelta = dictMessage.isDelta
+
+      // 2. The dictionary data is actually just a RecordBatch with ONE column
+      // The schema for this internal batch is defined by the dictionary type
+      // found in the global Schema for this specific ID.
+      guard let rbMessage = dictMessage.data else {
+        throw ArrowError(.invalid("DictionaryBatch has no data"))
+      }
+
+      let dictBatch = try Self.loadRecordBatch(
+        data: self.data,
+        arrowSchema: arrowSchema,
+        rbMessage: rbMessage,
+        offset: bodyOffset
+      )
+
+      // 4. Update the "Box" in your provider
+      //        try dictionaryProvider.update(id: dictId, array: dictionaryArray, isDelta: isDelta)
+    }
+
     // MARK: Record batch parsing
     for block in footer.recordBatches {
 

Sources/ArrowIPC/FlatBuffersTypeAliases.swift

@@ -13,16 +13,22 @@
 // limitations under the License.
 
 typealias FFooter = org_apache_arrow_flatbuf_Footer
+
+typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader
 typealias FMessage = org_apache_arrow_flatbuf_Message
+
 typealias FBlock = org_apache_arrow_flatbuf_Block
 typealias FField = org_apache_arrow_flatbuf_Field
 typealias FSchema = org_apache_arrow_flatbuf_Schema
 typealias FBuffer = org_apache_arrow_flatbuf_Buffer
 typealias FFieldNode = org_apache_arrow_flatbuf_FieldNode
-typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch
-typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader
 typealias FKeyValue = org_apache_arrow_flatbuf_KeyValue
 
+// MARK: Record batches.
+typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch
+typealias FDictionaryBatch = org_apache_arrow_flatbuf_DictionaryBatch
+typealias FDictionaryEncoding = org_apache_arrow_flatbuf_DictionaryEncoding
+
 // MARK: Top level type.
 typealias FType = org_apache_arrow_flatbuf_Type_
 

Tests/ArrowIPCTests/ArrowTestingGold.swift

@@ -82,10 +82,10 @@ struct ArrowTestingGold {
     "generated_union",
   ]
 
-  @Test
+//  @Test
   func readFile() throws {
 
-    let name = "generated_nested_large_offsets"
+    let name = "generated_dictionary"
     let (testFile, testCase) = try loadTestCase(
       name: name, fileExtension: "arrow_file")
     let arrowReader = try ArrowReader(url: testFile)