Skip to content

Commit 7114c85

Browse files
authored
feat(sqlframe): Allow options in scan_parquet function (#2295)
* feat(sqlframe): Allow options in scan_parquet * version parsing fix
1 parent 57f9cc0 commit 7114c85

File tree

2 files changed

+6
-5
lines changed

2 files changed

+6
-5
lines changed

narwhals/functions.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import platform
44
import sys
5+
from importlib.metadata import version
56
from typing import TYPE_CHECKING
67
from typing import Any
78
from typing import Iterable
@@ -1138,12 +1139,12 @@ def _scan_parquet_impl(
11381139
if (session := kwargs.pop("session", None)) is None:
11391140
msg = "Spark like backends require a session object to be passed in `kwargs`."
11401141
raise ValueError(msg)
1141-
11421142
native_frame = (
11431143
session.read.format("parquet").load(source)
1144-
# passing `options` currently not possible in SQLFrame: see
1145-
# https://github.com/eakmanrq/sqlframe/issues/341
1146-
if implementation is Implementation.SQLFRAME
1144+
if (
1145+
implementation is Implementation.SQLFRAME
1146+
and (parse_version(version("sqlframe"))) < (3, 27, 0)
1147+
)
11471148
else session.read.format("parquet").options(**kwargs).load(source)
11481149
)
11491150

tests/read_scan_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ def test_scan_parquet(
158158
if "sqlframe" in str(constructor):
159159
from sqlframe.duckdb import DuckDBSession
160160

161-
kwargs = {"session": DuckDBSession()}
161+
kwargs = {"session": DuckDBSession(), "inferSchema": True}
162162
elif "pyspark" in str(constructor):
163163
from pyspark.sql import SparkSession
164164

0 commit comments

Comments
 (0)