Use Arrow Type Fixed-Width Binary ("w:") For Fixed-Width TILEDB_CHAR (#1286)

* Previously, this errored out with `ArrowInvalid: Expected 3 buffers
  for imported type large_binary, ArrowArray struct has 2`
* Fixed-width `TILEDB_CHAR` was using type large binary ("Z") which
  requires 3 buffers, including an offset buffer, that is not present
  or necessary for fixed-width binary strings
* This does not handle fixed-width `TILEDB_UTF8` or `TILEDB_ASCII`

Author: nguyenv

HISTORY.md

@@ -1,7 +1,8 @@
 # In Progress
 
 ## Bug Fixes
-* Fix issue where querying an array with a Boolean type when `arrow=True`, but is unselected in `.query(attr=...)`, results in an error `pyarrow.lib.ArrowInvalid: Invalid column index to set field.` []()
+* Fix issue where querying an array with a Boolean type when `arrow=True`, but is unselected in `.query(attr=...)`, results in an error `pyarrow.lib.ArrowInvalid: Invalid column index to set field.` [#1291](https://github.com/TileDB-Inc/TileDB-Py/pull/1291)
+* Use Arrow type fixed-width binary ("w:") for non-variable TILEDB_CHAR [#1286](https://github.com/TileDB-Inc/TileDB-Py/pull/1286)
 
 # TileDB-Py 0.17.1 Release Notes
 

tiledb/py_arrow_io_impl.h

@@ -183,7 +183,9 @@ ArrowInfo tiledb_buffer_arrow_fmt(BufferInfo bufferinfo, bool use_list = true) {
       return ArrowInfo("U");
     }
   case TILEDB_CHAR:
-    if (bufferinfo.offsets_elem_size == 4) {
+    if (!bufferinfo.is_var) {
+      return ArrowInfo("w:" + std::to_string(cell_val_num));
+    } else if (bufferinfo.offsets_elem_size == 4) {
       return ArrowInfo("z");
     } else {
       return ArrowInfo("Z");
@@ -304,7 +306,10 @@ TypeInfo arrow_type_to_tiledb(ArrowSchema *arw_schema) {
     return {TILEDB_DATETIME_NS, 8, 1, large};
   else if (fmt == "z" || fmt == "Z")
     return {TILEDB_CHAR, 1, TILEDB_VAR_NUM, fmt == "Z"};
-  else if (fmt == "u" || fmt == "U")
+  else if (fmt.rfind("w:", 0) == 0) {
+    uint32_t cell_val_num = atoi(fmt.substr(2).c_str());
+    return {TILEDB_CHAR, 1, cell_val_num, fmt == "w"};
+  } else if (fmt == "u" || fmt == "U")
     return {TILEDB_STRING_UTF8, 1, TILEDB_VAR_NUM, fmt == "U"};
 #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 10
   else if (fmt == "b")
@@ -769,7 +774,14 @@ void ArrowExporter::export_(const std::string &name, ArrowArray *array,
     // adjust for arrow offset unless empty result
     elem_num = (bufferinfo.offsets_num == 0) ? 0 : bufferinfo.offsets_num - 1;
   } else {
-    elem_num = bufferinfo.data_num;
+    if (arrow_fmt.fmt_.rfind("w:", 0) == 0) {
+      // for Arrow fixed-width binary (non-variable TILEDB_CHAR), we need to
+      // take the size of the entire buffer and divide by the size of each
+      // element
+      elem_num = bufferinfo.data_num / bufferinfo.tdbtype.cell_val_num;
+    } else {
+      elem_num = bufferinfo.data_num;
+    }
   }
 
   auto cpp_arrow_array = new CPPArrowArray(elem_num, // elem_num

tiledb/tests/test_multi_index.py

@@ -888,6 +888,23 @@ def test_multi_index_timing(self):
             assert "py.getitem_time.pandas_index_update_time :" in internal_stats
         tiledb.stats_disable()
 
+    @pytest.mark.skipif(not has_pandas(), reason="pandas not installed")
+    def test_fixed_width_char(self):
+        uri = self.path("test_fixed_width_char")
+        schema = tiledb.ArraySchema(
+            domain=tiledb.Domain(tiledb.Dim(name="dim", domain=(0, 2), dtype=np.uint8)),
+            attrs=[tiledb.Attr(dtype="|S3")],
+        )
+        tiledb.Array.create(uri, schema)
+
+        data = np.array(["cat", "dog", "hog"], dtype="|S3")
+
+        with tiledb.open(uri, mode="w") as A:
+            A[:] = data
+
+        with tiledb.open(uri, mode="r") as A:
+            assert all(A.query(use_arrow=True).df[:][""] == data)
+
 
 # parametrize dtype and sparse
 @pytest.mark.parametrize(