ARROW-113 Treat nan as null when coercing to int (#152)

Remove pandas -0b1 << 63 workaround.

Author: ShaneHarvey

.github/workflows/test-python.yml

@@ -87,7 +87,7 @@ jobs:
           python -m pip install -v -e ".[test]"
       - name: Run the tests
         run: |
-          PYTHONWARNINGS=error python -m pytest -vv
+          PYTHONWARNINGS=error python -m pytest -s -vv
       - name: Check the manifest
         run: |
           pip install check-manifest

bindings/python/pymongoarrow/lib.pyx

@@ -21,6 +21,7 @@ import copy
 import datetime
 import enum
 import sys
+from math import isnan
 
 # Python imports
 import bson
@@ -269,9 +270,15 @@ cdef void process_raw_bson_stream(const uint8_t * docstream, size_t length, obje
                     int64_builder = builder
                     if (value_t == BSON_TYPE_INT64 or
                             value_t == BSON_TYPE_BOOL or
-                            value_t == BSON_TYPE_DOUBLE or
                             value_t == BSON_TYPE_INT32):
                         int64_builder.append_raw(bson_iter_as_int64(&doc_iter))
+                    elif value_t == BSON_TYPE_DOUBLE:
+                        # Treat nan as null.
+                        val = bson_iter_as_double(&doc_iter)
+                        if isnan(val):
+                            int64_builder.append_null()
+                        else:
+                            int64_builder.append_raw(bson_iter_as_int64(&doc_iter))
                     else:
                         int64_builder.append_null()
                 elif ftype == BSON_TYPE_OID:

bindings/python/test/test_pandas.py

@@ -354,8 +354,8 @@ def find_fn(self, coll, query, schema):
         return find_pandas_all(coll, query, schema=schema)
 
     def equal_fn(self, left, right):
-        left = left.fillna(0).replace(-0b1 << 63, 0)  # NaN is sometimes this
-        right = right.fillna(0).replace(-0b1 << 63, 0)
+        left = left.fillna(0)
+        right = right.fillna(0)
         if type(left) == pandas.DataFrame:
             pandas.testing.assert_frame_equal(left, right, check_dtype=False)
         else: