Skip to content

Commit 6b93548

Browse files
authored
Fixing nested struct bug in athena2pyarrow method (#612)
1 parent 6be3e65 commit 6b93548

File tree

1 file changed

+8
-4
lines changed

1 file changed

+8
-4
lines changed

awswrangler/_data_types.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -251,8 +251,10 @@ def _split_map(s: str) -> List[str]:
251251
return parts
252252

253253

254-
def athena2pyarrow(dtype: str) -> pa.DataType: # pylint: disable=too-many-return-statements
254+
def athena2pyarrow(dtype: str) -> pa.DataType: # pylint: disable=too-many-return-statements,too-many-branches
255255
"""Athena to PyArrow data types conversion."""
256+
if dtype.startswith(("array", "struct", "map")):
257+
orig_dtype: str = dtype
256258
dtype = dtype.lower().replace(" ", "")
257259
if dtype == "tinyint":
258260
return pa.int8()
@@ -280,11 +282,13 @@ def athena2pyarrow(dtype: str) -> pa.DataType: # pylint: disable=too-many-retur
280282
precision, scale = dtype.replace("decimal(", "").replace(")", "").split(sep=",")
281283
return pa.decimal128(precision=int(precision), scale=int(scale))
282284
if dtype.startswith("array") is True:
283-
return pa.list_(value_type=athena2pyarrow(dtype=dtype[6:-1]), list_size=-1)
285+
return pa.list_(value_type=athena2pyarrow(dtype=orig_dtype[6:-1]), list_size=-1)
284286
if dtype.startswith("struct") is True:
285-
return pa.struct([(f.split(":", 1)[0], athena2pyarrow(f.split(":", 1)[1])) for f in _split_struct(dtype[7:-1])])
287+
return pa.struct(
288+
[(f.split(":", 1)[0], athena2pyarrow(f.split(":", 1)[1])) for f in _split_struct(orig_dtype[7:-1])]
289+
)
286290
if dtype.startswith("map") is True:
287-
parts: List[str] = _split_map(s=dtype[4:-1])
291+
parts: List[str] = _split_map(s=orig_dtype[4:-1])
288292
return pa.map_(athena2pyarrow(parts[0]), athena2pyarrow(parts[1]))
289293
raise exceptions.UnsupportedType(f"Unsupported Athena type: {dtype}")
290294

0 commit comments

Comments
 (0)