fix: use from_arrow instead of from_arrow_refs (#2417)

Signed-off-by: Abdel Jaidi <[email protected]>

Author: jaidisido

awswrangler/distributed/ray/modin/_utils.py

@@ -68,7 +68,8 @@ def _is_not_empty(table: pa.Table) -> Any:
     ref_rows: List[bool] = ray_get([_is_not_empty(arrow_ref) for arrow_ref in arrow_refs])
     refs: List[Callable[..., Any]] = [ref for ref_rows, ref in zip(ref_rows, arrow_refs) if ref_rows]
     return _to_modin(
-        dataset=ray.data.from_arrow_refs(refs if len(refs) > 0 else [pa.Table.from_arrays([])]), to_pandas_kwargs=kwargs
+        dataset=ray.data.from_arrow_refs(refs) if len(refs) > 0 else ray.data.from_arrow([pa.Table.from_arrays([])]),
+        to_pandas_kwargs=kwargs,
     )
 
 

tests/unit/test_s3_select.py

@@ -67,11 +67,6 @@ def test_full_table(path, use_threads):
     assert df.shape == df4.shape
 
 
-@pytest.mark.xfail(
-    raises=AssertionError,
-    reason="https://github.com/ray-project/ray/issues/37928",
-    condition=is_ray_modin,
-)
 @pytest.mark.parametrize("use_threads", [True, False, 2])
 def test_push_down(path, use_threads):
     df = pd.DataFrame({"c0": [1, 2, 3], "c1": ["foo", "boo", "bar"], "c2": [4.0, 5.0, 6.0]})