fixup timezones

fixup timezone handling, by Marco

Author: FBruzzesi

packages/python/plotly/_plotly_utils/basevalidators.py

@@ -73,8 +73,6 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     """
     np = get_module("numpy")
 
-    # Don't force pandas to be loaded, we only want to know if it's already loaded
-    pd = get_module("pandas", should_load=False)
     assert np is not None
 
     # ### Process kind ###
@@ -94,45 +92,26 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
         "O": "object",
     }
 
+    # With `pass_through=True``, the original object will be returned if unable to convert
+    # to a Narwhals DataFrame or Series.
+    v = nw.from_native(v, allow_series=True, pass_through=True)
+
     if isinstance(v, nw.Series):
-        if nw.dependencies.is_pandas_like_series(v_native := v.to_native()):
-            v = v_native
+        if v.dtype == nw.Datetime and v.dtype.time_zone is not None:
+            # Remove time zone so that local time is displayed
+            v = v.dt.replace_time_zone(None).to_numpy()
         else:
             v = v.to_numpy()
     elif isinstance(v, nw.DataFrame):
-        if nw.dependencies.is_pandas_like_dataframe(v_native := v.to_native()):
-            v = v_native
-        else:
-            v = v.to_numpy()
-
-    if pd and isinstance(v, (pd.Series, pd.Index)):
-        # Handle pandas Series and Index objects
-        if v.dtype.kind in numeric_kinds:
-            # Get the numeric numpy array so we use fast path below
-            v = v.values
-        elif v.dtype.kind == "M":
-            # Convert datetime Series/Index to numpy array of datetimes
-            if isinstance(v, pd.Series):
-                with warnings.catch_warnings():
-                    warnings.simplefilter("ignore", FutureWarning)
-                    # Series.dt.to_pydatetime will return Index[object]
-                    # https://github.com/pandas-dev/pandas/pull/52459
-                    v = np.array(v.dt.to_pydatetime())
-            else:
-                # DatetimeIndex
-                v = v.to_pydatetime()
-    elif pd and isinstance(v, pd.DataFrame) and len(set(v.dtypes)) == 1:
-        dtype = v.dtypes.tolist()[0]
-        if dtype.kind in numeric_kinds:
-            v = v.values
-        elif dtype.kind == "M":
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore", FutureWarning)
-                # Series.dt.to_pydatetime will return Index[object]
-                # https://github.com/pandas-dev/pandas/pull/52459
-                v = [
-                    np.array(row.dt.to_pydatetime()).tolist() for i, row in v.iterrows()
-                ]
+        schema = v.schema
+        overrides = {}
+        for key, val in schema.items():
+            if val == nw.Datetime and val.time_zone is not None:
+                # Remove time zone so that local time is displayed
+                overrides[key] = nw.col(key).dt.replace_time_zone(None)
+        if overrides:
+            v = v.with_columns(**overrides)
+        v = v.to_numpy()
 
     if not isinstance(v, np.ndarray):
         # v has its own logic on how to convert itself into a numpy array

packages/python/plotly/optional-requirements.txt

@@ -39,7 +39,7 @@ ipython
 
 ## pandas deps for some matplotlib functionality ##
 pandas
-narwhals>=1.12.0
+narwhals>=1.13.1
 
 ## scipy deps for some FigureFactory functions ##
 scipy

packages/python/plotly/plotly/express/_core.py

@@ -413,12 +413,20 @@ def make_trace_kwargs(args, trace_spec, trace_data, mapping_labels, sizeref):
                     # i.e. we can't do resampling, because then the X values might not line up!
                     non_missing = ~(x.is_null() | y.is_null())
                     trace_patch["x"] = (
-                        sorted_trace_data.filter(non_missing)
-                        .get_column(args["x"])
-                        .to_numpy()
+                        sorted_trace_data.filter(non_missing).get_column(args["x"])
                         # FIXME: Converting to numpy is needed to pass `test_trendline_on_timeseries`
                         # test, but I wonder if it is the right way to do it in the first place.
                     )
+                    if (
+                        trace_patch["x"].dtype == nw.Datetime
+                        and trace_patch["x"].dtype.time_zone is not None
+                    ):
+                        # Remove time zone so that local time is displayed
+                        trace_patch["x"] = (
+                            trace_patch["x"].dt.replace_time_zone(None).to_numpy()
+                        )
+                    else:
+                        trace_patch["x"] = trace_patch["x"].to_numpy()
 
                     trendline_function = trendline_functions[attr_value]
                     y_out, hover_header, fit_results = trendline_function(
@@ -1166,7 +1174,7 @@ def to_unindexed_series(x, name=None, native_namespace=None):
     its index reset if pandas-like). Stripping the index from existing pd.Series is
     required to get things to match up right in the new DataFrame we're building.
     """
-    x = nw.from_native(x, series_only=True, strict=False)
+    x = nw.from_native(x, series_only=True, pass_through=True)
     if isinstance(x, nw.Series):
         return nw.maybe_reset_index(x).rename(name)
     elif native_namespace is not None:
@@ -1372,7 +1380,7 @@ def process_args_into_dataframe(
                     )
 
                 df_output[str(col_name)] = to_unindexed_series(
-                    x=nw.from_native(argument, series_only=True, strict=False),
+                    x=nw.from_native(argument, series_only=True, pass_through=True),
                     name=str(col_name),
                     native_namespace=native_namespace,
                 )
@@ -1500,11 +1508,11 @@ def build_dataframe(args, constructor):
             is_pd_like = True
 
         # data_frame is any other DataFrame object natively supported via Narwhals.
-        # With strict=False, the original object will be returned if unable to convert
+        # With pass_through=True, the original object will be returned if unable to convert
         # to a Narwhals DataFrame, making this condition False.
         elif isinstance(
             data_frame := nw.from_native(
-                args["data_frame"], eager_or_interchange_only=True, strict=False
+                args["data_frame"], eager_or_interchange_only=True, pass_through=True
             ),
             nw.DataFrame,
         ):
@@ -1513,11 +1521,11 @@ def build_dataframe(args, constructor):
             columns = args["data_frame"].columns
 
         # data_frame is any other Series object natively supported via Narwhals.
-        # With strict=False, the original object will be returned if unable to convert
-        # to a Narwhals DataFrame, making this condition False.
+        # With pass_through=True, the original object will be returned if unable to convert
+        # to a Narwhals Series, making this condition False.
         elif isinstance(
             series := nw.from_native(
-                args["data_frame"], series_only=True, strict=False
+                args["data_frame"], series_only=True, pass_through=True
             ),
             nw.Series,
         ):

packages/python/plotly/plotly/express/_imshow.py

@@ -321,7 +321,7 @@ def imshow(
             aspect = "equal"
 
     # --- Set the value of binary_string (forbidden for pandas)
-    img = nw.from_native(img, strict=False)
+    img = nw.from_native(img, pass_through=True)
     if isinstance(img, nw.DataFrame):
         if binary_string:
             raise ValueError("Binary strings cannot be used with pandas arrays")

packages/python/plotly/plotly/tests/test_optional/test_px/test_px_hover.py

@@ -190,14 +190,12 @@ def test_sunburst_hoverdict_color(constructor):
 
 
 def test_date_in_hover(request, constructor):
-    if "pyarrow_table" in str(constructor) or "polars_eager" in str(constructor):
-        # fig.data[0].customdata[0][0] is a numpy.datetime64 for non pandas
-        # input, and it does not keep the timezone when converting to py scalar
-        request.applymarker(pytest.mark.xfail)
-
     df = nw.from_native(
         constructor({"date": ["2015-04-04 19:31:30+01:00"], "value": [3]})
     ).with_columns(date=nw.col("date").str.to_datetime(format="%Y-%m-%d %H:%M:%S%z"))
     fig = px.scatter(df.to_native(), x="value", y="value", hover_data=["date"])
 
-    assert fig.data[0].customdata[0][0] == df.item(row=0, column="date")
+    # Check that what gets displayed is the local datetime
+    assert nw.to_py_scalar(fig.data[0].customdata[0][0]) == nw.to_py_scalar(
+        df.item(row=0, column="date")
+    ).replace(tzinfo=None)

packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py

@@ -270,7 +270,7 @@ def test_encode_customdata_datetime_series(self):
         )
         self.assertTrue(
             fig_json.startswith(
-                '{"data":[{"customdata":["2010-01-01T00:00:00","2010-01-02T00:00:00"]'
+                '{"data":[{"customdata":["2010-01-01T00:00:00.000000000","2010-01-02T00:00:00.000000000"]'
             )
         )
 
@@ -292,8 +292,8 @@ def test_encode_customdata_datetime_homogeneous_dataframe(self):
         self.assertTrue(
             fig_json.startswith(
                 '{"data":[{"customdata":'
-                '[["2010-01-01T00:00:00","2011-01-01T00:00:00"],'
-                '["2010-01-02T00:00:00","2011-01-02T00:00:00"]'
+                '[["2010-01-01T00:00:00.000000000","2011-01-01T00:00:00.000000000"],'
+                '["2010-01-02T00:00:00.000000000","2011-01-02T00:00:00.000000000"]'
             )
         )
 

packages/python/plotly/requirements.txt

@@ -6,4 +6,4 @@
 ###################################################
 
 ## dataframe agnostic layer ##
-narwhals>=1.12.0
+narwhals>=1.13.1

packages/python/plotly/setup.py

@@ -603,7 +603,7 @@ def run(self):
     data_files=[
         ("etc/jupyter/nbconfig/notebook.d", ["jupyterlab-plotly.json"]),
     ],
-    install_requires=["narwhals>=1.12.0", "packaging"],
+    install_requires=["narwhals>=1.13.1", "packaging"],
     zip_safe=False,
     cmdclass=dict(
         build_py=js_prerelease(versioneer_cmds["build_py"]),

packages/python/plotly/test_requirements/requirements_310_core.txt

@@ -1,3 +1,3 @@
 requests==2.25.1
 pytest==7.4.4
-narwhals>=1.12.0
+narwhals>=1.13.1

packages/python/plotly/test_requirements/requirements_310_optional.txt

@@ -21,4 +21,4 @@ kaleido
 orjson==3.8.12
 polars[timezone]
 pyarrow
-narwhals>=1.12.0
+narwhals>=1.13.1