feat: support DataFrames via narwhals

pyproject.toml

@@ -47,6 +47,8 @@ dependencies = [
     "prompt-toolkit;platform_system!='Emscripten'",
     "python-multipart>=0.0.7;platform_system!='Emscripten'",
     "setuptools;python_version>='3.12'",
+    "narwhals>=1.1.7",
+    "orjson>=3.10.7",
 ]
 
 [project.optional-dependencies]

shiny/render/_data_frame_utils/_html.py

@@ -9,6 +9,8 @@
 from ._types import CellHtml, ReprHtml, SeriesLike
 
 if TYPE_CHECKING:
+    import narwhals.stable.v1 as nw
+
     from ...session import Session
 
 
@@ -41,8 +43,11 @@ def maybe_as_cell_html(
     return cast(Jsonifiable, x)
 
 
-def col_contains_shiny_html(col: SeriesLike) -> bool:
-    return any(is_shiny_html(val) for _, val in enumerate(col))
+def col_contains_shiny_html(col: SeriesLike | nw.Series) -> bool:
+    for val in col:
+        if is_shiny_html(val):
+            return True
+    return False
 
 
 # TODO-barret-test; Add test to assert the union type of `TagNode` contains `str` and (HTML | Tagifiable | MetadataNode | ReprHtml). Until a `is tag renderable` method is available in htmltools, we need to check for these types manually and must stay in sync with the `TagNode` union type.

shiny/render/_data_frame_utils/_pandas.py

@@ -80,6 +80,9 @@ def wrap_shiny_html_with_session(x: TagNode):
 
     res["typeHints"] = type_hints
 
+    if "index" in res:
+        del res["index"]
+
     # print(json.dumps(res, indent=4))
     return res
 
@@ -119,8 +122,10 @@ def serialize_pd_dtype(
         }
     elif t in {"datetime64", "datetime"}:
         t = "datetime"
-    elif t in {"timedelta", "timedelta64"}:
-        t = "timedelta"
+
+    # # Disable timedelta as narwhals does not support it
+    # elif t in {"timedelta", "timedelta64"}:
+    #     t = "timedelta"
     else:
         if col_contains_shiny_html(col):
             t = "html"

shiny/render/_data_frame_utils/_tbl_data.py

@@ -5,13 +5,12 @@
 from functools import singledispatch
 from typing import Any, List, Tuple, cast
 
+import narwhals.stable.v1 as nw
 from htmltools import TagNode
 
 from ..._typing_extensions import TypeIs
 from ...session import require_active_session
 from ._html import maybe_as_cell_html
-
-# from ...types import Jsonifiable
 from ._types import (
     CellPatch,
     ColsList,
@@ -184,7 +183,7 @@ def _(col: PlSeries) -> FrameDtype:
 
     from ._html import col_contains_shiny_html
 
-    if col.dtype.is_(pl.String):
+    if col.dtype == pl.String():
         if col_contains_shiny_html(col):
             type_ = "html"
         else:
@@ -203,6 +202,58 @@ def _(col: PlSeries) -> FrameDtype:
     return {"type": type_}
 
 
+nw_boolean = nw.Boolean()
+nw_categorical = nw.Categorical()
+nw_enum = nw.Enum()
+nw_string = nw.String()
+nw_datetime = nw.Datetime()
+nw_duration = nw.Duration()
+nw_object = nw.Object()
+
+
+@serialize_dtype.register
+def _(col: nw.Series) -> FrameDtype:
+
+    from ._html import col_contains_shiny_html
+
+    if col.dtype == nw_string:
+        if col_contains_shiny_html(col):
+            type_ = "html"
+        else:
+            type_ = "string"
+    elif col.dtype.is_numeric():
+        type_ = "numeric"
+
+    elif col.dtype == nw_categorical:
+        categories = col.cat.get_categories().to_list()
+        return {"type": "categorical", "categories": categories}
+    elif col.dtype == nw_enum:
+        raise NotImplementedError("boolean type not tested")
+        cat_col = col.cast(nw.Categorical)
+        categories = cat_col.cat.get_categories().to_list()
+        return {"type": "categorical", "categories": categories}
+
+    elif col.dtype == nw_boolean:
+        raise NotImplementedError("boolean type not tested")
+        type_ = "boolean"
+    elif col.dtype == nw_duration:
+        raise NotImplementedError("duration type not tested")
+        type_ = "duration"
+    elif col.dtype == nw_datetime:
+        type_ = "datetime"
+    elif col.dtype == nw_object:
+        type_ = "object"
+        if col_contains_shiny_html(col):
+            type_ = "html"
+
+    else:
+        type_ = "unknown"
+        if col_contains_shiny_html(col):
+            type_ = "html"
+
+    return {"type": type_}
+
+
 # serialize_frame ----------------------------------------------------------------------
 
 
@@ -218,6 +269,7 @@ def _(data: PdDataFrame) -> FrameJson:
     return serialize_frame_pd(data)
 
 
+# TODO: test this
 @serialize_frame.register
 def _(data: PlDataFrame) -> FrameJson:
     import json
@@ -249,6 +301,61 @@ def wrap_shiny_html_with_session(x: TagNode):
     }
 
 
+@serialize_frame.register(nw.DataFrame)
+def _(data: nw.DataFrame[Any]) -> FrameJson:
+
+    type_hints = [serialize_dtype(data[col_name]) for col_name in data.columns]
+    type_hints_type = {type_hint["type"] for type_hint in type_hints}
+
+    data_rows = data.rows(named=False)
+
+    # print(data_rows)
+    # print(data.rows(named=False))
+
+    # Shiny tag support
+    if "html" in type_hints_type:
+        session = require_active_session(None)
+
+        def wrap_shiny_html_with_session(x: TagNode):
+            return maybe_as_cell_html(x, session=session)
+
+        html_column_positions = [
+            i for i, x in enumerate(type_hints_type) if x == "html"
+        ]
+
+        new_rows: list[tuple[Any, ...]] = []
+
+        # Wrap the corresponding columns with the cell html object
+        for row in data_rows:
+            new_row = list(row)
+            for html_column_position in html_column_positions:
+                new_row[html_column_position] = wrap_shiny_html_with_session(
+                    new_row[html_column_position]
+                )
+            new_rows.append(tuple(new_row))
+
+        data_rows = new_rows
+
+    import orjson
+
+    data_val = orjson.loads(
+        orjson.dumps(
+            data_rows,
+            default=str,
+            option=orjson.OPT_UTC_Z,
+        )
+    )
+
+    # import json
+    # data_val = json.loads(json.dumps(data_rows, default=str))
+
+    return {
+        "columns": data.columns,
+        "data": data_val,
+        "typeHints": type_hints,
+    }
+
+
 # subset_frame -------------------------------------------------------------------------
 def subset_frame__typed(
     data: DataFrameLikeT, *, rows: RowsList = None, cols: ColsList = None
@@ -308,6 +415,7 @@ def _(
         return data.iloc[indx_rows, indx_cols]
 
 
+@subset_frame.register(nw.DataFrame)
 @subset_frame.register
 def _(
     data: PlDataFrame,
@@ -321,7 +429,7 @@ def _(
         else slice(None)
     )
     indx_rows = rows if rows is not None else slice(None)
-    return data[indx_rows, indx_cols]
+    return data[indx_cols][indx_rows]
 
 
 # get_frame_cell -----------------------------------------------------------------------
@@ -341,9 +449,10 @@ def _(data: PdDataFrame, row: int, col: int) -> Any:
     )
 
 
+@get_frame_cell.register(nw.DataFrame)
 @get_frame_cell.register
 def _(data: PlDataFrame, row: int, col: int) -> Any:
-    return data[row, col]
+    return data.item(row, col)
 
 
 # shape --------------------------------------------------------------------------------
@@ -359,6 +468,7 @@ def _(data: PdDataFrame) -> Tuple[int, ...]:
     return data.shape
 
 
+@frame_shape.register(nw.DataFrame)
 @frame_shape.register
 def _(data: PlDataFrame) -> Tuple[int, ...]:
     return data.shape
@@ -377,6 +487,7 @@ def _(data: PdDataFrame) -> PdDataFrame:
     return data.copy()
 
 
+@copy_frame.register(nw.DataFrame)
 @copy_frame.register
 def _(data: PlDataFrame) -> PlDataFrame:
     return data.clone()
@@ -393,6 +504,7 @@ def _(data: PdDataFrame) -> List[str]:
     return data.columns.to_list()
 
 
+@frame_column_names.register(nw.DataFrame)
 @frame_column_names.register
 def _(data: PlDataFrame) -> List[str]:
     return data.columns

shiny/render/_data_frame_utils/_types.py

@@ -70,6 +70,7 @@ class DataFrameLike(ABC): ...
     DataFrameLike.register(PlDataFrame)
 
 DataFrameLikeT = TypeVar("DataFrameLikeT", PdDataFrame, PlDataFrame)
+SeriesLikeT = TypeVar("SeriesLikeT", PdSeries, PlSeries)
 
 # ---------------------------------------------------------------------
 
@@ -175,7 +176,17 @@ class FrameJson(TypedDict):
 
 
 class FrameDtypeSubset(TypedDict):
-    type: Literal["numeric", "string", "html", "datetime", "timedelta", "unknown"]
+    type: Literal[
+        "boolean",
+        "numeric",
+        "string",
+        "html",
+        "datetime",
+        # "timedelta",
+        "duration",
+        "object",
+        "unknown",
+    ]
 
 
 class FrameDtypeCategories(TypedDict):