INTPYTHON-575 Fix handling of empty embedded arrays (#296)

Author: blink1073

bindings/python/pymongoarrow/context.py

@@ -75,7 +75,7 @@ def _parse_builder_map(builder_map):
         elif value.type_marker == _BsonArrowTypes.array.value:
             child_name = key + "[]"
             to_remove.append(child_name)
-            child = builder_map[child_name]
+            child = builder_map.get(child_name, [])
             builder_map[key] = ListArray.from_arrays(value.finish(), child)
         else:
             builder_map[key] = value.finish()

bindings/python/test/__init__.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import pymongo
 
 
@@ -42,4 +43,3 @@ def init(self):
 
 
 client_context = ClientContext()
-client_context.init()

bindings/python/test/conftest.py

@@ -1,4 +1,4 @@
-# Copyright 2023-present MongoDB, Inc.
+# Copyright 2025-present MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,59 +11,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import numpy as np
-import pandas as pd
-import pytest
-
-# Fixtures for use with Pandas extension types.
-
-
-@pytest.fixture
-def data_for_twos(dtype):
-    return pd.array(np.ones(100), dtype=dtype)
-
-
-@pytest.fixture
-def na_value():
-    return np.nan
-
-
-@pytest.fixture
-def na_cmp():
-    def cmp(a, b):
-        return np.isnan(a) and np.isnan(b)
-
-    return cmp
-
-
-@pytest.fixture(params=[True, False])
-def box_in_series(request):
-    """Whether to box the data in a Series"""
-    return request.param
-
-
-@pytest.fixture(params=[True, False])
-def as_array(request):
-    """
-    Boolean fixture to support ExtensionDtype _from_sequence method testing.
-    """
-    return request.param
+from test import client_context
 
+import pytest
 
-@pytest.fixture(params=["ffill", "bfill"])
-def fillna_method(request):
-    """
-    Parametrized fixture giving method parameters 'ffill' and 'bfill' for
-    Series.fillna(method=<method>) testing.
-    """
-    return request.param
+pytest_plugins = [
+    "pandas.tests.extension.conftest",
+]
 
 
-@pytest.fixture
-def invalid_scalar(data):
-    """
-    A scalar that *cannot* be held by this ExtensionArray.
-    The default should work for most subclasses, but is not guaranteed.
-    If the array can hold any item (i.e. object dtype), then use pytest.skip.
-    """
-    return object.__new__(object)
+@pytest.fixture(autouse=True, scope="session")
+def client():
+    client_context.init()
+    yield
+    if client_context.client:
+        client_context.client.close()

bindings/python/test/test_arrow.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import io
 import json
 import tempfile
 import unittest
@@ -22,8 +23,9 @@
 from test.utils import AllowListEventListener, NullsTestMixin
 
 import pyarrow as pa
+import pyarrow.json
 import pymongo
-from bson import Binary, Code, CodecOptions, Decimal128, ObjectId
+from bson import Binary, Code, CodecOptions, Decimal128, ObjectId, json_util
 from pyarrow import (
     Table,
     bool_,
@@ -1021,6 +1023,35 @@ def test_decimal128(self):
         coll_data = list(self.coll.find({}))
         assert coll_data[0]["data"] == Decimal128(a)
 
+    def test_empty_embedded_array(self):
+        # From INTPYTHON-575.
+        self.coll.drop()
+
+        self.coll.insert_many(
+            [{"_id": 1, "foo": {"bar": ["1", "2"]}}, {"_id": 2, "foo": {"bar": []}}]
+        )
+
+        # get document out of mongo, put it in a file and read it with pyarrow and write it to parquet.
+        doc1 = self.coll.find_one({"_id": 1})
+        string1 = json_util.dumps(doc1, indent=2)
+        file1 = io.BytesIO(bytes(string1, encoding="utf-8"))
+        papatable1 = pyarrow.json.read_json(file1)
+        write_table(papatable1, io.BytesIO())
+
+        # read document with pymongoarrow and write it to parquet.
+        pmapatable1 = find_arrow_all(self.coll, {"_id": {"$eq": 1}})
+        write_table(pmapatable1, io.BytesIO())
+
+        doc2 = self.coll.find_one({"_id": 2})
+        string2 = json_util.dumps(doc2, indent=2)
+        file2 = io.BytesIO(bytes(string2, encoding="utf-8"))
+        papatable2 = pyarrow.json.read_json(file2)
+        write_table(papatable2, io.BytesIO())
+
+        pmapatable2 = find_arrow_all(self.coll, {"_id": {"$eq": 2}})
+        assert pmapatable2.to_pylist()[0] == doc2
+        write_table(pmapatable2, io.BytesIO())
+
 
 class TestArrowExplicitApi(ArrowApiTestMixin, unittest.TestCase):
     def run_find(self, *args, **kwargs):

bindings/python/test/utils.py

@@ -144,6 +144,10 @@ def setUp(self):
         self.cmd_listener.reset()
         self.getmore_listener.reset()
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
     def assertType(self, obj1, arrow_type):
         if isinstance(obj1, pa.ChunkedArray):
             if "storage_type" in dir(arrow_type) and obj1.type != arrow_type: