diff --git a/IPython/core/display_trap.py b/IPython/core/display_trap.py index 9931dfe2dfc..a32d8759784 100644 --- a/IPython/core/display_trap.py +++ b/IPython/core/display_trap.py @@ -58,6 +58,10 @@ def __exit__(self, type, value, traceback): # Returning False will cause exceptions to propagate return False + @property + def is_active(self) -> bool: + return self._nested_level != 0 + def set(self): """Set the hook.""" if sys.displayhook is not self.hook: diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index b1de9df7ee1..ee6dbe8f477 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -16,6 +16,8 @@ from traitlets import Instance, Float from warnings import warn +from .history import HistoryOutput + # TODO: Move the various attributes (cache_size, [others now moved]). Some # of these are also attributes of InteractiveShell. They should be on ONE object # only and the other objects should ask that one object for their values. @@ -35,6 +37,7 @@ class DisplayHook(Configurable): def __init__(self, shell=None, cache_size=1000, **kwargs): super(DisplayHook, self).__init__(shell=shell, **kwargs) + self._is_active = False cache_size_min = 3 if cache_size <= 0: self.do_full_cache = 0 @@ -51,7 +54,7 @@ def __init__(self, shell=None, cache_size=1000, **kwargs): # we need a reference to the user-level namespace self.shell = shell - + self._,self.__,self.___ = '','','' # these are deliberately global: @@ -84,13 +87,13 @@ def check_for_underscore(self): def quiet(self): """Should we silence the display hook because of ';'?""" # do not print output if input ends in ';' - + try: cell = self.shell.history_manager.input_hist_parsed[-1] except IndexError: # some uses of ipshellembed may fail here return False - + return self.semicolon_at_end_of_expression(cell) @staticmethod @@ -110,7 +113,11 @@ def semicolon_at_end_of_expression(expression): def start_displayhook(self): """Start the displayhook, initializing resources.""" - pass + self._is_active = True + + @property + def is_active(self): + return self._is_active def write_output_prompt(self): """Write the output prompt. @@ -242,7 +249,10 @@ def fill_exec_result(self, result): def log_output(self, format_dict): """Log the output.""" - if 'text/plain' not in format_dict: + self.shell.history_manager.outputs[self.prompt_count].append( + HistoryOutput(output_type="execute_result", bundle=format_dict) + ) + if "text/plain" not in format_dict: # nothing to do return if self.shell.logger.log_output: @@ -254,6 +264,7 @@ def finish_displayhook(self): """Finish up all displayhook activities.""" sys.stdout.write(self.shell.separate_out2) sys.stdout.flush() + self._is_active = False def __call__(self, result=None): """Printing with history cache management. @@ -280,13 +291,12 @@ def cull_cache(self): cull_count = max(int(sz * self.cull_fraction), 2) warn('Output cache limit (currently {sz} entries) hit.\n' 'Flushing oldest {cull_count} entries.'.format(sz=sz, cull_count=cull_count)) - + for i, n in enumerate(sorted(oh)): if i >= cull_count: break self.shell.user_ns.pop('_%i' % n, None) oh.pop(n, None) - def flush(self): if not self.do_full_cache: diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index 1428c68ea78..41c633fb143 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -22,6 +22,7 @@ # This used to be defined here - it is imported for backwards compatibility from .display_functions import publish_display_data +from .history import HistoryOutput import typing as t @@ -41,6 +42,7 @@ class DisplayPublisher(Configurable): def __init__(self, shell=None, *args, **kwargs): self.shell = shell + self._is_publishing = False super().__init__(*args, **kwargs) def _validate_data(self, data, metadata=None): @@ -129,13 +131,25 @@ def publish( if self.shell is not None: handlers = getattr(self.shell, "mime_renderers", {}) + outputs = self.shell.history_manager.outputs + + outputs[self.shell.execution_count].append( + HistoryOutput(output_type="display_data", bundle=data) + ) + for mime, handler in handlers.items(): if mime in data: handler(data[mime], metadata.get(mime, None)) return + self._is_publishing = True if "text/plain" in data: print(data["text/plain"]) + self._is_publishing = False + + @property + def is_publishing(self): + return self._is_publishing def clear_output(self, wait=False): """Clear the output of the cell receiving output.""" diff --git a/IPython/core/history.py b/IPython/core/history.py index cdfd25760a7..abb434407c9 100644 --- a/IPython/core/history.py +++ b/IPython/core/history.py @@ -14,7 +14,9 @@ import threading from pathlib import Path +from collections import defaultdict from contextlib import contextmanager +from dataclasses import dataclass from decorator import decorator from traitlets import ( Any, @@ -583,6 +585,14 @@ def get_range_by_str( yield from self.get_range(sess, s, e, raw=raw, output=output) +@dataclass +class HistoryOutput: + output_type: typing.Literal[ + "out_stream", "err_stream", "display_data", "execute_result" + ] + bundle: typing.Dict[str, str] + + class HistoryManager(HistoryAccessor): """A class to organize all history-related functionality in one place.""" @@ -610,7 +620,11 @@ def _dir_hist_default(self) -> list[Path]: # execution count. output_hist = Dict() # The text/plain repr of outputs. - output_hist_reprs: dict[int, str] = Dict() # type: ignore [assignment] + output_hist_reprs: typing.Dict[int, str] = Dict() # type: ignore [assignment] + # Maps execution_count to MIME bundles + outputs: typing.Dict[int, typing.List[HistoryOutput]] = defaultdict(list) + # Maps execution_count to exception tracebacks + exceptions: typing.Dict[int, typing.Dict[str, Any]] = Dict() # type: ignore [assignment] # The number of the current session in the history database session_number: int = Integer() # type: ignore [assignment] @@ -749,6 +763,9 @@ def reset(self, new_session: bool = True) -> None: """Clear the session history, releasing all object references, and optionally open a new session.""" self.output_hist.clear() + self.outputs.clear() + self.exceptions.clear() + # The directory history can't be completely empty self.dir_hist[:] = [Path.cwd()] diff --git a/IPython/core/historyapp.py b/IPython/core/historyapp.py index d555447b5d4..220ae9ad96e 100644 --- a/IPython/core/historyapp.py +++ b/IPython/core/historyapp.py @@ -59,9 +59,9 @@ def start(self): print("There are already at most %d entries in the history database." % self.keep) print("Not doing anything. Use --keep= argument to keep fewer entries") return - + print("Trimming history to the most recent %d entries." % self.keep) - + inputs.pop() # Remove the extra element we got to check the length. inputs.reverse() if inputs: @@ -71,7 +71,7 @@ def start(self): sessions = list(con.execute('SELECT session, start, end, num_cmds, remark FROM ' 'sessions WHERE session >= ?', (first_session,))) con.close() - + # Create the new history database. new_hist_file = profile_dir / "history.sqlite.new" i = 0 diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 67cca752d25..9cc7023246a 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -28,12 +28,13 @@ import types import warnings from ast import stmt +from contextlib import contextmanager from io import open as io_open from logging import error from pathlib import Path from typing import Callable from typing import List as ListType, Any as AnyType -from typing import Optional, Sequence, Tuple +from typing import Literal, Optional, Sequence, Tuple from warnings import warn from IPython.external.pickleshare import PickleShareDB @@ -71,7 +72,7 @@ from IPython.core.events import EventManager, available_events from IPython.core.extensions import ExtensionManager from IPython.core.formatters import DisplayFormatter -from IPython.core.history import HistoryManager +from IPython.core.history import HistoryManager, HistoryOutput from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 from IPython.core.logger import Logger from IPython.core.macro import Macro @@ -324,6 +325,7 @@ def _modified_open(file, *args, **kwargs): return io_open(file, *args, **kwargs) + class InteractiveShell(SingletonConfigurable): """An enhanced, interactive shell for Python.""" @@ -660,6 +662,7 @@ def __init__(self, ipython_dir=None, profile_dir=None, # inside a single Trio event loop. If used, it is set from # `ipykernel.kernelapp`. self.trio_runner = None + self.showing_traceback = False @property def user_ns(self): @@ -865,7 +868,7 @@ def init_displayhook(self): cache_size=self.cache_size, ) self.configurables.append(self.displayhook) - # This is a context manager that installs/revmoes the displayhook at + # This is a context manager that installs/removes the displayhook at # the appropriate time. self.display_trap = DisplayTrap(hook=self.displayhook) @@ -2197,10 +2200,12 @@ def _showtraceback(self, etype, evalue, stb: str): place, like a side channel. """ val = self.InteractiveTB.stb2text(stb) + self.showing_traceback = True try: print(val) except UnicodeEncodeError: print(val.encode("utf-8", "backslashreplace").decode()) + self.showing_traceback = False def showsyntaxerror(self, filename=None, running_compiled_code=False): """Display the syntax error that just occurred. @@ -3003,6 +3008,51 @@ def safe_run_module(self, mod_name, where): self.showtraceback() warn('Unknown failure executing module: <%s>' % mod_name) + @contextmanager + def _tee(self, channel: Literal["stdout", "stderr"]): + """Capture output of a given standard stream and store it in history. + + Uses patching of write method for maximal compatibility, + because ipykernel checks for instances of the stream class, + and stream classes in ipykernel implement more complex logic. + """ + stream = getattr(sys, channel) + original_write = stream.write + + def write(data, *args, **kwargs): + """Write data to both the original destination and the capture dictionary.""" + result = original_write(data, *args, **kwargs) + if any( + [ + self.display_pub.is_publishing, + self.displayhook.is_active, + self.showing_traceback, + ] + ): + return result + if not data: + return result + execution_count = self.execution_count + output_stream = None + outputs_by_counter = self.history_manager.outputs + output_type = "out_stream" if channel == "stdout" else "err_stream" + if execution_count in outputs_by_counter: + outputs = outputs_by_counter[execution_count] + if outputs[-1].output_type == output_type: + output_stream = outputs[-1] + if output_stream is None: + output_stream = HistoryOutput( + output_type=output_type, bundle={"stream": ""} + ) + outputs_by_counter[execution_count].append(output_stream) + + output_stream.bundle["stream"] += data # Append to existing stream + return result + + stream.write = write + yield + stream.write = original_write + def run_cell( self, raw_cell, @@ -3035,14 +3085,15 @@ def run_cell( result : :class:`ExecutionResult` """ result = None - try: - result = self._run_cell( - raw_cell, store_history, silent, shell_futures, cell_id - ) - finally: - self.events.trigger('post_execute') - if not silent: - self.events.trigger('post_run_cell', result) + with self._tee(channel="stdout"), self._tee(channel="stderr"): + try: + result = self._run_cell( + raw_cell, store_history, silent, shell_futures, cell_id + ) + finally: + self.events.trigger("post_execute") + if not silent: + self.events.trigger("post_run_cell", result) return result def _run_cell( @@ -3200,6 +3251,11 @@ async def run_cell_async( def error_before_exec(value): if store_history: + if self.history_manager: + # Store formatted traceback and error details + self.history_manager.exceptions[self.execution_count] = ( + self._format_exception_for_storage(value) + ) self.execution_count += 1 result.error_before_exec = value self.last_execution_succeeded = False @@ -3310,11 +3366,73 @@ def error_before_exec(value): # Write output to the database. Does nothing unless # history output logging is enabled. self.history_manager.store_output(self.execution_count) + exec_count = self.execution_count + if result.error_in_exec: + # Store formatted traceback and error details + self.history_manager.exceptions[exec_count] = ( + self._format_exception_for_storage(result.error_in_exec) + ) + # Each cell is a *single* input, regardless of how many lines it has self.execution_count += 1 return result + def _format_exception_for_storage( + self, exception, filename=None, running_compiled_code=False + ): + """ + Format an exception's traceback and details for storage, with special handling + for different types of errors. + """ + etype = type(exception) + evalue = exception + tb = exception.__traceback__ + + # Handle SyntaxError and IndentationError with specific formatting + if issubclass(etype, (SyntaxError, IndentationError)): + if filename and isinstance(evalue, SyntaxError): + try: + evalue.filename = filename + except: + pass # Keep the original filename if modification fails + + # Extract traceback if the error happened during compiled code execution + elist = traceback.extract_tb(tb) if running_compiled_code else [] + stb = self.SyntaxTB.structured_traceback(etype, evalue, elist) + + # Handle UsageError with a simple message + elif etype is UsageError: + stb = [f"UsageError: {evalue}"] + + else: + # Check if the exception (or its context) is an ExceptionGroup. + def contains_exceptiongroup(val): + if val is None: + return False + return isinstance(val, BaseExceptionGroup) or contains_exceptiongroup( + val.__context__ + ) + + if contains_exceptiongroup(evalue): + # Fallback: use the standard library's formatting for exception groups. + stb = traceback.format_exception(etype, evalue, tb) + else: + try: + # If the exception has a custom traceback renderer, use it. + if hasattr(evalue, "_render_traceback_"): + stb = evalue._render_traceback_() + else: + # Otherwise, use InteractiveTB to format the traceback. + stb = self.InteractiveTB.structured_traceback( + etype, evalue, tb, tb_offset=1 + ) + except Exception: + # In case formatting fails, fallback to Python's built-in formatting. + stb = traceback.format_exception(etype, evalue, tb) + + return {"ename": etype.__name__, "evalue": str(evalue), "traceback": stb} + def transform_cell(self, raw_cell): """Transform an input cell before parsing it. diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index d68020075b1..ec1ab532337 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -21,7 +21,7 @@ class MagicsDisplay: def __init__(self, magics_manager, ignore=None): self.ignore = ignore if ignore else [] self.magics_manager = magics_manager - + def _lsmagic(self): """The main implementation of the %lsmagic""" mesc = magic_escapes['line'] @@ -39,13 +39,13 @@ def _lsmagic(self): def _repr_pretty_(self, p, cycle): p.text(self._lsmagic()) - + def __repr__(self): return self.__str__() def __str__(self): return self._lsmagic() - + def _jsonable(self): """turn magics dict into jsonable dict of the same structure @@ -62,10 +62,10 @@ def _jsonable(self): classname = obj.__self__.__class__.__name__ except AttributeError: classname = 'Other' - + d[name] = classname return magic_dict - + def _repr_json_(self): return self._jsonable() @@ -561,13 +561,46 @@ def notebook(self, s): cells = [] hist = list(self.shell.history_manager.get_range()) + outputs = self.shell.history_manager.outputs + exceptions = self.shell.history_manager.exceptions + if(len(hist)<=1): raise ValueError('History is empty, cannot export') for session, execution_count, source in hist[:-1]: - cells.append(v4.new_code_cell( - execution_count=execution_count, - source=source - )) + cell = v4.new_code_cell(execution_count=execution_count, source=source) + for output in outputs[execution_count]: + for mime_type, data in output.bundle.items(): + if output.output_type == "out_stream": + cell.outputs.append(v4.new_output("stream", text=[data])) + elif output.output_type == "err_stream": + err_output = v4.new_output("stream", text=[data]) + err_output.name = "stderr" + cell.outputs.append(err_output) + elif output.output_type == "execute_result": + cell.outputs.append( + v4.new_output( + "execute_result", + data={mime_type: data}, + execution_count=execution_count, + ) + ) + elif output.output_type == "display_data": + cell.outputs.append( + v4.new_output( + "display_data", + data={mime_type: data}, + ) + ) + else: + raise ValueError(f"Unknown output type: {output.output_type}") + + # Check if this execution_count is in exceptions (current session) + if execution_count in exceptions: + cell.outputs.append( + v4.new_output("error", **exceptions[execution_count]) + ) + cells.append(cell) + nb = v4.new_notebook(cells=cells) with io.open(outfname, "w", encoding="utf-8") as f: write(nb, f, version=4) diff --git a/pyproject.toml b/pyproject.toml index 0e58d321c28..f722ea31c08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,8 @@ test_extra = [ "curio", "matplotlib!=3.2.0", "nbformat", + "nbclient", + "ipykernel", "numpy>=1.23", "pandas", "trio", diff --git a/tests/test_magic.py b/tests/test_magic.py index a2e6f80e5ee..af1fc3ecf26 100644 --- a/tests/test_magic.py +++ b/tests/test_magic.py @@ -381,9 +381,7 @@ def test_reset_in_length(): class TestResetErrors(TestCase): - def test_reset_redefine(self): - @magics_class class KernelMagics(Magics): @line_magic @@ -914,8 +912,71 @@ def test_notebook_export_json(): _ip.run_line_magic("notebook", "%s" % outfile) -class TestEnv(TestCase): +def test_notebook_export_json_with_output(): + """Tests if notebook export correctly captures outputs, errors, display outputs, and stream outputs.""" + pytest.importorskip("nbformat") + pytest.importorskip("nbclient") + import nbformat + from nbclient import NotebookClient + _ip = get_ipython() + _ip.history_manager.reset() + _ip.colors = "neutral" + _ip.execution_count = 1 + + try: + commands = [ + "1/0", + "print('test')", + "display('test')", + "1+1", + "display('a'), display('b')", + "import sys\nprint('test', file=sys.stderr)", + ] + + clean_nb = nbformat.v4.new_notebook( + cells=[nbformat.v4.new_code_cell(source=cmd) for cmd in commands] + ) + + with TemporaryDirectory() as td: + outfile = os.path.join(td, "nb.ipynb") + client = NotebookClient( + clean_nb, + timeout=600, + kernel_name="python3", + resources={"metadata": {"path": td}}, + allow_errors=True, + ) + client.execute() + nbformat.write(clean_nb, outfile) + expected_nb = nbformat.read(outfile, as_version=4) + + for cmd in commands: + _ip.run_cell(cmd, store_history=True, silent=False) + print(f"\n{_ip.history_manager.outputs}\n") + + with TemporaryDirectory() as td: + outfile = os.path.join(td, "nb.ipynb") + _ip.run_cell(f"%notebook {outfile}", store_history=True) + sleep(2) + actual_nb = nbformat.read(outfile, as_version=4) + + assert len(actual_nb["cells"]) == len(commands) + assert len(expected_nb["cells"]) == len(commands) + + for i, command in enumerate(commands): + actual = actual_nb["cells"][i] + expected = expected_nb["cells"][i] + assert expected["source"] == command + assert actual["source"] == expected["source"] + assert ( + actual["outputs"] == expected["outputs"] + ), f"Outputs do not match for cell {i+1} with source {command!r}" + finally: + _ip.colors = "nocolor" + + +class TestEnv(TestCase): def test_env(self): env = _ip.run_line_magic("env", "") self.assertTrue(isinstance(env, dict))