Bug: ValueError when using opensearch.index_df with documents with an array field (#1444)

* cast df values to numpy array to evaulate notna(v).all()

* notna 'all' -> 'any'

* adding test

* formatting

Co-authored-by: kukushking <[email protected]>

Author: malachi-constant

awswrangler/opensearch/_write.py

@@ -7,6 +7,7 @@
 from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple, Union
 
 import boto3
+import numpy as np
 import pandas as pd
 import progressbar
 from jsonpath_ng import parse
@@ -79,7 +80,7 @@ def _deserialize(v: Any) -> Any:
 
     df_iter = df.iterrows()
     for _, document in df_iter:
-        yield {k: _deserialize(v) for k, v in document.items() if notna(v)}
+        yield {k: _deserialize(v) for k, v in document.items() if np.array(notna(v)).any()}
 
 
 def _file_line_generator(path: str, is_json: bool = False) -> Generator[Any, None, None]:

tests/test_opensearch.py

@@ -215,6 +215,18 @@ def test_index_df(client):
     assert response.get("success", 0) == 3
 
 
+def test_index_df_with_array(client):
+    response = wr.opensearch.index_df(
+        client,
+        df=pd.DataFrame(
+            [{"_id": "1", "name": "John", "tags": ["foo", "bar"]}, {"_id": "2", "name": "George", "tags": ["foo"]}]
+        ),
+        index="test_index_df1",
+    )
+    print(response)
+    assert response.get("success", 0) == 2
+
+
 def test_index_documents(client):
     response = wr.opensearch.index_documents(
         client,