Merge branch 'main' into 60237

Author: road-1

doc/source/whatsnew/v3.0.0.rst

@@ -703,6 +703,8 @@ I/O
 - Bug in :meth:`read_stata` raising ``KeyError`` when input file is stored in big-endian format and contains strL data. (:issue:`58638`)
 - Bug in :meth:`read_stata` where extreme value integers were incorrectly interpreted as missing for format versions 111 and prior (:issue:`58130`)
 - Bug in :meth:`read_stata` where the missing code for double was not recognised for format versions 105 and prior (:issue:`58149`)
+- Bug in :meth:`set_option` where setting the pandas option ``display.html.use_mathjax`` to ``False`` has no effect (:issue:`59884`)
+- Bug in :meth:`to_excel` where :class:`MultiIndex` columns would be merged to a single row when ``merge_cells=False`` is passed (:issue:`60274`)
 
 Period
 ^^^^^^

pandas/core/dtypes/cast.py

@@ -1162,6 +1162,7 @@ def convert_dtypes(
 
 def maybe_infer_to_datetimelike(
     value: npt.NDArray[np.object_],
+    convert_to_nullable_dtype: bool = False,
 ) -> np.ndarray | DatetimeArray | TimedeltaArray | PeriodArray | IntervalArray:
     """
     we might have a array (or single object) that is datetime like,
@@ -1199,6 +1200,7 @@ def maybe_infer_to_datetimelike(
         #  numpy would have done it for us.
         convert_numeric=False,
         convert_non_numeric=True,
+        convert_to_nullable_dtype=convert_to_nullable_dtype,
         dtype_if_all_nat=np.dtype("M8[s]"),
     )
 

pandas/core/frame.py

@@ -2115,8 +2115,8 @@ def from_records(
         """
         Convert structured or record ndarray to DataFrame.
 
-        Creates a DataFrame object from a structured ndarray, sequence of
-        tuples or dicts, or DataFrame.
+        Creates a DataFrame object from a structured ndarray, or sequence of
+        tuples or dicts.
 
         Parameters
         ----------

pandas/core/generic.py

@@ -2211,8 +2211,9 @@ def to_excel(
             via the options ``io.excel.xlsx.writer`` or
             ``io.excel.xlsm.writer``.
 
-        merge_cells : bool, default True
-            Write MultiIndex and Hierarchical Rows as merged cells.
+        merge_cells : bool or 'columns', default False
+            If True, write MultiIndex index and columns as merged cells.
+            If 'columns', merge MultiIndex column cells only.
         {encoding_parameter}
         inf_rep : str, default 'inf'
             Representation for infinity (there is no native representation for

pandas/core/internals/blocks.py

@@ -108,6 +108,7 @@
     PeriodArray,
     TimedeltaArray,
 )
+from pandas.core.arrays.string_ import StringDtype
 from pandas.core.base import PandasObject
 import pandas.core.common as com
 from pandas.core.computation import expressions
@@ -1336,7 +1337,7 @@ def fillna(
             return [self.copy(deep=False)]
 
         if limit is not None:
-            mask[mask.cumsum(self.ndim - 1) > limit] = False
+            mask[mask.cumsum(self.values.ndim - 1) > limit] = False
 
         if inplace:
             nbs = self.putmask(mask.T, value)
@@ -1684,7 +1685,7 @@ def where(self, other, cond) -> list[Block]:
             res_values = arr._where(cond, other).T
         except (ValueError, TypeError):
             if self.ndim == 1 or self.shape[0] == 1:
-                if isinstance(self.dtype, IntervalDtype):
+                if isinstance(self.dtype, (IntervalDtype, StringDtype)):
                     # TestSetitemFloatIntervalWithIntIntervalValues
                     blk = self.coerce_to_target_dtype(orig_other, raise_on_upcast=False)
                     return blk.where(orig_other, orig_cond)
@@ -1854,9 +1855,9 @@ def fillna(
         limit: int | None = None,
         inplace: bool = False,
     ) -> list[Block]:
-        if isinstance(self.dtype, IntervalDtype):
+        if isinstance(self.dtype, (IntervalDtype, StringDtype)):
             # Block.fillna handles coercion (test_fillna_interval)
-            if limit is not None:
+            if isinstance(self.dtype, IntervalDtype) and limit is not None:
                 raise ValueError("limit must be None")
             return super().fillna(
                 value=value,

pandas/core/internals/construction.py

@@ -966,8 +966,9 @@ def convert(arr):
             if dtype is None:
                 if arr.dtype == np.dtype("O"):
                     # i.e. maybe_convert_objects didn't convert
-                    arr = maybe_infer_to_datetimelike(arr)
-                    if dtype_backend != "numpy" and arr.dtype == np.dtype("O"):
+                    convert_to_nullable_dtype = dtype_backend != "numpy"
+                    arr = maybe_infer_to_datetimelike(arr, convert_to_nullable_dtype)
+                    if convert_to_nullable_dtype and arr.dtype == np.dtype("O"):
                         new_dtype = StringDtype()
                         arr_cls = new_dtype.construct_array_type()
                         arr = arr_cls._from_sequence(arr, dtype=new_dtype)

pandas/io/formats/excel.py

@@ -48,7 +48,6 @@
     CSSWarning,
 )
 from pandas.io.formats.format import get_level_lengths
-from pandas.io.formats.printing import pprint_thing
 
 if TYPE_CHECKING:
     from pandas._typing import (
@@ -620,61 +619,43 @@ def _format_header_mi(self) -> Iterable[ExcelCell]:
             return
 
         columns = self.columns
-        level_strs = columns._format_multi(
-            sparsify=self.merge_cells in {True, "columns"}, include_names=False
-        )
+        merge_columns = self.merge_cells in {True, "columns"}
+        level_strs = columns._format_multi(sparsify=merge_columns, include_names=False)
         level_lengths = get_level_lengths(level_strs)
         coloffset = 0
         lnum = 0
 
         if self.index and isinstance(self.df.index, MultiIndex):
             coloffset = self.df.index.nlevels - 1
 
-        if self.merge_cells in {True, "columns"}:
-            # Format multi-index as a merged cells.
-            for lnum, name in enumerate(columns.names):
-                yield ExcelCell(
-                    row=lnum,
-                    col=coloffset,
-                    val=name,
-                    style=None,
-                )
+        for lnum, name in enumerate(columns.names):
+            yield ExcelCell(
+                row=lnum,
+                col=coloffset,
+                val=name,
+                style=None,
+            )
 
-            for lnum, (spans, levels, level_codes) in enumerate(
-                zip(level_lengths, columns.levels, columns.codes)
-            ):
-                values = levels.take(level_codes)
-                for i, span_val in spans.items():
-                    mergestart, mergeend = None, None
-                    if span_val > 1:
-                        mergestart, mergeend = lnum, coloffset + i + span_val
-                    yield CssExcelCell(
-                        row=lnum,
-                        col=coloffset + i + 1,
-                        val=values[i],
-                        style=None,
-                        css_styles=getattr(self.styler, "ctx_columns", None),
-                        css_row=lnum,
-                        css_col=i,
-                        css_converter=self.style_converter,
-                        mergestart=mergestart,
-                        mergeend=mergeend,
-                    )
-        else:
-            # Format in legacy format with dots to indicate levels.
-            for i, values in enumerate(zip(*level_strs)):
-                v = ".".join(map(pprint_thing, values))
+        for lnum, (spans, levels, level_codes) in enumerate(
+            zip(level_lengths, columns.levels, columns.codes)
+        ):
+            values = levels.take(level_codes)
+            for i, span_val in spans.items():
+                mergestart, mergeend = None, None
+                if merge_columns and span_val > 1:
+                    mergestart, mergeend = lnum, coloffset + i + span_val
                 yield CssExcelCell(
                     row=lnum,
                     col=coloffset + i + 1,
-                    val=v,
+                    val=values[i],
                     style=None,
                     css_styles=getattr(self.styler, "ctx_columns", None),
                     css_row=lnum,
                     css_col=i,
                     css_converter=self.style_converter,
+                    mergestart=mergestart,
+                    mergeend=mergeend,
                 )
-
         self.rowcounter = lnum
 
     def _format_header_regular(self) -> Iterable[ExcelCell]:
@@ -798,11 +779,8 @@ def _format_hierarchical_rows(self) -> Iterable[ExcelCell]:
 
             # MultiIndex columns require an extra row
             # with index names (blank if None) for
-            # unambiguous round-trip, unless not merging,
-            # in which case the names all go on one row Issue #11328
-            if isinstance(self.columns, MultiIndex) and (
-                self.merge_cells in {True, "columns"}
-            ):
+            # unambiguous round-trip, Issue #11328
+            if isinstance(self.columns, MultiIndex):
                 self.rowcounter += 1
 
             # if index labels are not empty go ahead and dump

pandas/io/formats/html.py

@@ -241,6 +241,7 @@ def _write_table(self, indent: int = 0) -> None:
         use_mathjax = get_option("display.html.use_mathjax")
         if not use_mathjax:
             _classes.append("tex2jax_ignore")
+            _classes.append("mathjax_ignore")
         if self.classes is not None:
             if isinstance(self.classes, str):
                 self.classes = self.classes.split()

pandas/io/formats/style_render.py

@@ -366,9 +366,11 @@ def _translate(
         if not get_option("styler.html.mathjax"):
             table_attr = table_attr or ""
             if 'class="' in table_attr:
-                table_attr = table_attr.replace('class="', 'class="tex2jax_ignore ')
+                table_attr = table_attr.replace(
+                    'class="', 'class="tex2jax_ignore mathjax_ignore '
+                )
             else:
-                table_attr += ' class="tex2jax_ignore"'
+                table_attr += ' class="tex2jax_ignore mathjax_ignore"'
         d.update({"table_attributes": table_attr})
 
         if self.tooltips:

pandas/io/sql.py

@@ -45,6 +45,8 @@
 from pandas.core.dtypes.common import (
     is_dict_like,
     is_list_like,
+    is_object_dtype,
+    is_string_dtype,
 )
 from pandas.core.dtypes.dtypes import (
     ArrowDtype,
@@ -58,6 +60,7 @@
     Series,
 )
 from pandas.core.arrays import ArrowExtensionArray
+from pandas.core.arrays.string_ import StringDtype
 from pandas.core.base import PandasObject
 import pandas.core.common as com
 from pandas.core.common import maybe_make_list
@@ -1316,7 +1319,12 @@ def _harmonize_columns(
                 elif dtype_backend == "numpy" and col_type is float:
                     # floats support NA, can always convert!
                     self.frame[col_name] = df_col.astype(col_type)
-
+                elif (
+                    using_string_dtype()
+                    and is_string_dtype(col_type)
+                    and is_object_dtype(self.frame[col_name])
+                ):
+                    self.frame[col_name] = df_col.astype(col_type)
                 elif dtype_backend == "numpy" and len(df_col) == df_col.count():
                     # No NA values, can convert ints and bools
                     if col_type is np.dtype("int64") or col_type is bool:
@@ -1403,6 +1411,7 @@ def _get_dtype(self, sqltype):
             DateTime,
             Float,
             Integer,
+            String,
         )
 
         if isinstance(sqltype, Float):
@@ -1422,6 +1431,10 @@ def _get_dtype(self, sqltype):
             return date
         elif isinstance(sqltype, Boolean):
             return bool
+        elif isinstance(sqltype, String):
+            if using_string_dtype():
+                return StringDtype(na_value=np.nan)
+
         return object
 
 
@@ -2205,7 +2218,7 @@ def read_table(
         elif using_string_dtype():
             from pandas.io._util import arrow_string_types_mapper
 
-            arrow_string_types_mapper()
+            mapping = arrow_string_types_mapper()
         else:
             mapping = None
 
@@ -2286,6 +2299,10 @@ def read_query(
             from pandas.io._util import _arrow_dtype_mapping
 
             mapping = _arrow_dtype_mapping().get
+        elif using_string_dtype():
+            from pandas.io._util import arrow_string_types_mapper
+
+            mapping = arrow_string_types_mapper()
         else:
             mapping = None