Workaround Arrow Table Incorrect Boolean Results (#1167)

* `TILEDB_BOOL` is represented as a `uint8_t` whereas Arrow's Boolean type is 1 bit
* Use Arrow's uint8 type in `tiledb_buffer_arrow_fmt` for `TILEDB_BOOL`
* Cast Boolean types in the resulting Arrow table in `_run_query` from uint8 to bool

Author: nguyenv

HISTORY.md

@@ -10,6 +10,7 @@
 
 ## Bug Fixes
 * Fix error where passing a `Context` to `Group` would segfault intermittenly [#1165](https://github.com/TileDB-Inc/TileDB-Py/pull/1165)
+* Correct Boolean values when `use_arrow=True` [#1167](https://github.com/TileDB-Inc/TileDB-Py/pull/1167)
 
 # TileDB-Py 0.15.3 Release Notes
 

tiledb/multirange_indexing.py

@@ -345,8 +345,21 @@ def _run_query(self) -> Union[DataFrame, Table]:
         elif self.use_arrow:
             with timing("buffer_conversion_time"):
                 table = self.pyquery._buffers_to_pa_table()
+
+            # this is a workaround to cast TILEDB_BOOL types from uint8
+            # representation in Arrow to Boolean
+            schema = table.schema
+            for n in range(self.array.nattr):
+                attr = self.array.attr(n)
+                if attr.dtype == bool:
+                    field_idx = schema.get_field_index(attr.name)
+                    field = pyarrow.field(attr.name, pyarrow.bool_())
+                    schema = schema.set(field_idx, field)
+            table = table.cast(schema)
+
             if self.query.return_arrow:
                 return table
+
             df = table.to_pandas()
         else:
             df = DataFrame(_get_pyquery_results(self.pyquery, self.array.schema))

tiledb/py_arrow_io_impl.h

@@ -229,8 +229,9 @@ ArrowInfo tiledb_buffer_arrow_fmt(BufferInfo bufferinfo, bool use_list = true) {
     return ArrowInfo("tsn:");
 
 #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 10
+  // TILEDB_BOOL is stored as a uint8_t but arrow::Type::BOOL is 1 bit
   case TILEDB_BOOL:
-    return ArrowInfo("b");
+    return ArrowInfo("C");
 #endif
 
   // TODO: these could potentially be rep'd w/ additional