BUG: to_json overflows when date does not fit in NS date. #60534
Description
Pandas version checks
-
I have checked that this issue has not already been reported.
-
I have confirmed this bug exists on the latest version of pandas.
-
I have confirmed this bug exists on the main branch of pandas.
Reproducible Example
import pandas as pd
import datetime as dt
df = pd.DataFrame(data={'date': [dt.datetime(2999, 1, 1)]}).to_json()Issue Description
When converting dataframes to json then dates that do not fit inside a NS precision timestamp will overflow.
---------------------------------------------------------------------------
OverflowError Traceback (most recent call last)
Cell In[46], [line 1](vscode-notebook-cell:?execution_count=46&line=1)
----> [1](vscode-notebook-cell:?execution_count=46&line=1) df = pd.DataFrame(data={'date': [dt.datetime(2999, 1, 1)]}).to_json()
File ~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:333, in deprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper(*args, **kwargs)
[327](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:327) if len(args) > num_allow_args:
[328](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:328) warnings.warn(
[329](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:329) msg.format(arguments=_format_argument_list(allow_args)),
[330](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:330) FutureWarning,
[331](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:331) stacklevel=find_stack_level(),
[332](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:332) )
--> [333](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/util/_decorators.py:333) return func(*args, **kwargs)
File ~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2702, in NDFrame.to_json(self, path_or_buf, orient, date_format, double_precision, force_ascii, date_unit, default_handler, lines, compression, index, indent, storage_options, mode)
[2699](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2699) config.is_nonnegative_int(indent)
[2700](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2700) indent = indent or 0
-> [2702](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2702) return json.to_json(
[2703](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2703) path_or_buf=path_or_buf,
[2704](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2704) obj=self,
[2705](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2705) orient=orient,
[2706](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2706) date_format=date_format,
[2707](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2707) double_precision=double_precision,
[2708](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2708) force_ascii=force_ascii,
[2709](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2709) date_unit=date_unit,
[2710](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2710) default_handler=default_handler,
[2711](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2711) lines=lines,
[2712](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2712) compression=compression,
[2713](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2713) index=index,
[2714](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2714) indent=indent,
[2715](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2715) storage_options=storage_options,
[2716](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2716) mode=mode,
[2717](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/core/generic.py:2717) )
File ~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:210, in to_json(path_or_buf, obj, orient, date_format, double_precision, force_ascii, date_unit, default_handler, lines, compression, index, indent, storage_options, mode)
[197](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:197) else:
[198](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:198) raise NotImplementedError("'obj' should be a Series or a DataFrame")
[200](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:200) s = writer(
[201](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:201) obj,
[202](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:202) orient=orient,
[203](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:203) date_format=date_format,
[204](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:204) double_precision=double_precision,
[205](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:205) ensure_ascii=force_ascii,
[206](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:206) date_unit=date_unit,
[207](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:207) default_handler=default_handler,
[208](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:208) index=index,
[209](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:209) indent=indent,
--> [210](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:210) ).write()
[212](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:212) if lines:
[213](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:213) s = convert_to_line_delimits(s)
File ~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:263, in Writer.write(self)
[261](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:261) def write(self) -> str:
[262](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:262) iso_dates = self.date_format == "iso"
--> [263](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:263) return ujson_dumps(
[264](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:264) self.obj_to_write,
[265](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:265) orient=self.orient,
[266](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:266) double_precision=self.double_precision,
[267](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:267) ensure_ascii=self.ensure_ascii,
[268](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:268) date_unit=self.date_unit,
[269](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:269) iso_dates=iso_dates,
[270](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:270) default_handler=self.default_handler,
[271](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:271) indent=self.indent,
[272](https://vscode-remote+ssh-002dremote-002bepyc-002dts.vscode-resource.vscode-cdn.net/home/skaae/code/datastores/~/code/datastores/.venv/lib/python3.12/site-packages/pandas/io/json/_json.py:272) )
OverflowError: Overflow occurred in npy_datetimestruct_to_datetime
Expected Behavior
Convert date to json
Installed Versions
INSTALLED VERSIONS
commit : 0691c5c
python : 3.12.3
python-bits : 64
OS : Linux
OS-release : 6.8.0-49-generic
Version : #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024
machine : x86_64
processor : x86_64
byteorder : little
LC_ALL : None
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8
pandas : 2.2.3
numpy : 1.26.4
pytz : 2024.2
dateutil : 2.9.0.post0
pip : 24.0
Cython : None
sphinx : None
IPython : 8.30.0
adbc-driver-postgresql: None
adbc-driver-sqlite : None
bs4 : 4.12.3
blosc : None
bottleneck : None
dataframe-api-compat : None
fastparquet : None
fsspec : None
html5lib : None
hypothesis : None
gcsfs : None
jinja2 : None
lxml.etree : None
matplotlib : 3.9.3
numba : 0.60.0
numexpr : None
odfpy : None
openpyxl : 3.1.5
pandas_gbq : None
psycopg2 : 2.9.10
pymysql : None
pyarrow : 18.1.0
pyreadstat : None
pytest : 8.3.4
python-calamine : None
pyxlsb : None
s3fs : None
scipy : 1.14.1
sqlalchemy : 2.0.36
tables : None
tabulate : None
xarray : 2024.11.0
xlrd : 2.0.1
xlsxwriter : None
zstandard : None
tzdata : 2024.2
qtpy : None
pyqt5 : None