Merge pull request #3084 from sinistersnare/bugfix/more-arrow-integers

Support a wider array of integers

Author: texodus

rust/perspective-python/perspective/tests/table/test_table_arrow.py

@@ -15,8 +15,11 @@
 import pandas as pd
 from perspective.tests.conftest import Util
 import pyarrow as pa
+import pyarrow.ipc as ipc
 from datetime import date, datetime
 import perspective as psp
+import io
+
 
 client = psp.Server().new_local_client()
 Table = client.table
@@ -25,11 +28,38 @@
 DATE64_ARROW = os.path.join(os.path.dirname(__file__), "arrow", "date64.arrow")
 DICT_ARROW = os.path.join(os.path.dirname(__file__), "arrow", "dict.arrow")
 
+
 names = ["a", "b", "c", "d"]
 
+# Create sample data for every integer type
+ALL_INTEGERS_DATA = {
+    "int8": pa.array([1, 2, 3], type=pa.int8()),
+    "int16": pa.array([1000, 2000, 3000], type=pa.int16()),
+    "int32": pa.array([100000, 200000, 300000], type=pa.int32()),
+    "int64": pa.array([10000000000, 20000000000, 30000000000], type=pa.int64()),
+    "uint8": pa.array([1, 2, 3], type=pa.uint8()),
+    "uint16": pa.array([1000, 2000, 3000], type=pa.uint16()),
+    "uint32": pa.array([100000, 200000, 300000], type=pa.uint32()),
+    "uint64": pa.array([10000000000, 20000000000, 30000000000], type=pa.uint64()),
+    "float32": pa.array([100000.0, 200000.0, 300000.0], type=pa.float32()),
+    "float64": pa.array(
+        [10000000000.0, 20000000000.0, 30000000000.0], type=pa.float64()
+    ),
+}
+
+ALL_INTEGERS_TABLE = pa.Table.from_pydict(ALL_INTEGERS_DATA)
+
+bytes_io = io.BytesIO()
+with ipc.new_stream(bytes_io, ALL_INTEGERS_TABLE.schema) as stream:
+    stream.write_table(ALL_INTEGERS_TABLE)
+ALL_INTEGERS_ARROW = bytes_io.getvalue()
 
 class TestTableArrow(object):
-    # files
+    def test_table_with_integer_types(self):
+        tbl = Table(ALL_INTEGERS_ARROW)
+        for k, values in ALL_INTEGERS_DATA.items():
+            v = tbl.view(filter=[[k, "==", values[0].as_py()]])
+            assert len(v.to_json()) == 1
 
     def test_table_arrow_loads_date32_file(self, util: Util):
         with open(DATE32_ARROW, mode="rb") as file:  # b is important -> binary

rust/perspective-python/requirements.txt

@@ -1,4 +1,5 @@
 pytest>=7.4.3
+maturin==1.6.0
 
 Faker==26.0.0
 ipywidgets==8.1.3

rust/perspective-server/cpp/perspective/src/cpp/server.cpp

@@ -1266,23 +1266,19 @@ coerce_to(const t_dtype dtype, const A& val) {
             case DTYPE_BOOL:
                 scalar.set(val == 1);
                 return scalar;
-            case DTYPE_UINT32:
-                scalar.set((std::uint32_t)val);
-                return scalar;
-            case DTYPE_UINT64:
-                scalar.set((std::uint64_t)val);
-                return scalar;
+            case DTYPE_UINT8:
+            case DTYPE_UINT16:
+            case DTYPE_INT8:
+            case DTYPE_INT16:
             case DTYPE_INT32:
-                scalar.set(val);
+                scalar.set(static_cast<std::int32_t>(val));
                 return scalar;
             case DTYPE_INT64:
-                scalar.set((std::int64_t)val);
-                return scalar;
+            case DTYPE_UINT32:
+            case DTYPE_UINT64:
             case DTYPE_FLOAT32:
-                scalar.set(static_cast<float>(val));
-                return scalar;
             case DTYPE_FLOAT64:
-                scalar.set(val);
+                scalar.set(static_cast<double>(val));
                 return scalar;
             case DTYPE_DATE: {
                 const auto time = static_cast<time_t>(val / 1000);
@@ -1302,7 +1298,9 @@ coerce_to(const t_dtype dtype, const A& val) {
                 return scalar;
             }
             default:
-                PSP_COMPLAIN_AND_ABORT("Unsupported double type");
+                std::stringstream ss;
+                ss << "Unsupported double type: " << dtype;
+                PSP_COMPLAIN_AND_ABORT(ss.str());
         }
     } else if constexpr (std::is_same_v<A, std::int32_t>) {
         return t_tscalar(val);