Combine individual lines of a multiline command into a single line for readline history.

Spaces and newlines in quotes are preserved so such history entries will display across multiple lines.
Non-verbose cmd2 history uses the same format as readline history entries.

Author: kmvanbrunt

CHANGELOG.md

@@ -3,6 +3,7 @@
   * `cmd2` 2.5 supports Python 3.8+ (removed support for Python 3.6 and 3.7)
 * Bug Fixes
   * Fixed issue where persistent history file was not saved upon SIGHUP and SIGTERM signals.
+  * Multiline commands are no longer fragmented in up-arrow history.
 * Enhancements
   * Removed dependency on `attrs` and replaced with [dataclasses](https://docs.python.org/3/library/dataclasses.html)
   * add `allow_clipboard` initialization parameter and attribute to disable ability to
@@ -12,7 +13,6 @@
 * Deletions (potentially breaking changes)
   * Removed `apply_style` from `Cmd.pwarning()`.
 
-
 ## 2.4.3 (January 27, 2023)
 * Bug Fixes
   * Fixed ValueError caused when passing `Cmd.columnize()` strings wider than `display_width`.

cmd2/cmd2.py

@@ -118,6 +118,7 @@
 from .history import (
     History,
     HistoryItem,
+    single_line_format,
 )
 from .parsing import (
     Macro,
@@ -2227,7 +2228,7 @@ def complete(  # type: ignore[override]
                 # Check if we are completing a multiline command
                 if self._at_continuation_prompt:
                     # lstrip and prepend the previously typed portion of this multiline command
-                    lstripped_previous = self._multiline_in_progress.lstrip().replace(constants.LINE_FEED, ' ')
+                    lstripped_previous = self._multiline_in_progress.lstrip()
                     line = lstripped_previous + readline.get_line_buffer()
 
                     # Increment the indexes to account for the prepended text
@@ -2503,7 +2504,13 @@ def parseline(self, line: str) -> Tuple[str, str, str]:
         return statement.command, statement.args, statement.command_and_args
 
     def onecmd_plus_hooks(
-        self, line: str, *, add_to_history: bool = True, raise_keyboard_interrupt: bool = False, py_bridge_call: bool = False
+        self,
+        line: str,
+        *,
+        add_to_history: bool = True,
+        raise_keyboard_interrupt: bool = False,
+        py_bridge_call: bool = False,
+        orig_rl_history_length: Optional[int] = None,
     ) -> bool:
         """Top-level function called by cmdloop() to handle parsing a line and running the command and all of its hooks.
 
@@ -2515,6 +2522,9 @@ def onecmd_plus_hooks(
         :param py_bridge_call: This should only ever be set to True by PyBridge to signify the beginning
                                of an app() call from Python. It is used to enable/disable the storage of the
                                command's stdout.
+        :param orig_rl_history_length: Optional length of the readline history before the current command was typed.
+                                       This is used to assist in combining multiline readline history entries and is only
+                                       populated by cmd2. Defaults to None.
         :return: True if running of commands should stop
         """
         import datetime
@@ -2524,7 +2534,7 @@ def onecmd_plus_hooks(
 
         try:
             # Convert the line into a Statement
-            statement = self._input_line_to_statement(line)
+            statement = self._input_line_to_statement(line, orig_rl_history_length=orig_rl_history_length)
 
             # call the postparsing hooks
             postparsing_data = plugin.PostparsingData(False, statement)
@@ -2738,11 +2748,14 @@ def _complete_statement(self, line: str) -> Statement:
             raise EmptyStatement
         return statement
 
-    def _input_line_to_statement(self, line: str) -> Statement:
+    def _input_line_to_statement(self, line: str, *, orig_rl_history_length: Optional[int] = None) -> Statement:
         """
         Parse the user's input line and convert it to a Statement, ensuring that all macros are also resolved
 
         :param line: the line being parsed
+        :param orig_rl_history_length: Optional length of the readline history before the current command was typed.
+                                       This is used to assist in combining multiline readline history entries and is only
+                                       populated by cmd2. Defaults to None.
         :return: parsed command line as a Statement
         :raises: Cmd2ShlexError if a shlex error occurs (e.g. No closing quotation)
         :raises: EmptyStatement when the resulting Statement is blank
@@ -2755,10 +2768,23 @@ def _input_line_to_statement(self, line: str) -> Statement:
             # Make sure all input has been read and convert it to a Statement
             statement = self._complete_statement(line)
 
-            # Save the fully entered line if this is the first loop iteration
+            # If this is the first loop iteration, save the original line and if necessary,
+            # combine multiline history entries into single readline history item.
             if orig_line is None:
                 orig_line = statement.raw
 
+                if orig_rl_history_length is not None and statement.multiline_command:
+                    # Remove all lines added to history for this command
+                    while readline.get_current_history_length() > orig_rl_history_length:
+                        readline.remove_history_item(readline.get_current_history_length() - 1)
+
+                    # Combine the lines of this command
+                    combined_command = single_line_format(statement)
+
+                    # If combined_command is different than the previous history item, then add it
+                    if orig_rl_history_length == 0 or combined_command != readline.get_history_item(orig_rl_history_length):
+                        readline.add_history(combined_command)
+
             # Check if this command matches a macro and wasn't already processed to avoid an infinite loop
             if statement.command in self.macros.keys() and statement.command not in used_macros:
                 used_macros.append(statement.command)
@@ -3111,7 +3137,7 @@ def configure_readline() -> None:
             nonlocal saved_history
             nonlocal parser
 
-            if readline_configured:  # pragma: no cover
+            if readline_configured or rl_type == RlType.NONE:  # pragma: no cover
                 return
 
             # Configure tab completion
@@ -3163,7 +3189,7 @@ def complete_none(text: str, state: int) -> Optional[str]:  # pragma: no cover
         def restore_readline() -> None:
             """Restore readline tab completion and history"""
             nonlocal readline_configured
-            if not readline_configured:  # pragma: no cover
+            if not readline_configured or rl_type == RlType.NONE:  # pragma: no cover
                 return
 
             if self._completion_supported():
@@ -3310,6 +3336,13 @@ def _cmdloop(self) -> None:
             self._startup_commands.clear()
 
             while not stop:
+                # Used in building multiline readline history entries. Only applies
+                # when command line is read using input() in a terminal.
+                if rl_type != RlType.NONE and self.use_rawinput and sys.stdin.isatty():
+                    orig_rl_history_length = readline.get_current_history_length()
+                else:
+                    orig_rl_history_length = None
+
                 # Get commands from user
                 try:
                     line = self._read_command_line(self.prompt)
@@ -3318,7 +3351,7 @@ def _cmdloop(self) -> None:
                     line = ''
 
                 # Run the command along with all associated pre and post hooks
-                stop = self.onecmd_plus_hooks(line)
+                stop = self.onecmd_plus_hooks(line, orig_rl_history_length=orig_rl_history_length)
         finally:
             # Get sigint protection while we restore readline settings
             with self.sigint_protection:
@@ -4873,13 +4906,13 @@ def _initialize_history(self, hist_file: str) -> None:
         if rl_type != RlType.NONE:
             last = None
             for item in self.history:
-                # Break the command into its individual lines
-                for line in item.raw.splitlines():
-                    # readline only adds a single entry for multiple sequential identical lines
-                    # so we emulate that behavior here
-                    if line != last:
-                        readline.add_history(line)
-                        last = line
+                formatted_command = single_line_format(item.statement)
+
+                # readline only adds a single entry for multiple sequential identical lines
+                # so we emulate that behavior here
+                if formatted_command != last:
+                    readline.add_history(formatted_command)
+                    last = formatted_command
 
     def _persist_history(self) -> None:
         """Write history out to the persistent history file as compressed JSON"""

cmd2/history.py

@@ -27,9 +27,51 @@
 )
 from .parsing import (
     Statement,
+    shlex_split,
 )
 
 
+def single_line_format(statement: Statement) -> str:
+    """
+    Format a command to display on a single line.
+
+    All spaces and newlines in quotes are preserved.
+
+    :param statement: Statement of multiline command being converted.
+    :return: converted multiline command as string
+    """
+    lines = statement.raw.rstrip().splitlines()
+    ret_str = lines[0]
+
+    # If first line doesn't have an unclosed quote, then remove trailing white space.
+    try:
+        shlex_split(ret_str)
+    except ValueError:
+        pass
+    else:
+        ret_str = ret_str.rstrip()
+
+    # Append any remaining lines to the command.
+    for line in lines[1:]:
+        try:
+            shlex_split(ret_str)
+        except ValueError:
+            # We are in quotes, so restore the newline.
+            separator = "\n"
+        else:
+            line = line.rstrip()
+
+            # Don't add a space if the line already begins with one or the terminator.
+            if line.startswith(" ") or line.startswith(statement.terminator):
+                separator = ""
+            else:
+                separator = " "
+
+        ret_str += separator + line
+
+    return ret_str
+
+
 @dataclass(frozen=True)
 class HistoryItem:
     """Class used to represent one command in the history list"""
@@ -85,15 +127,7 @@ def pr(self, idx: int, script: bool = False, expanded: bool = False, verbose: bo
             if expanded:
                 ret_str = self.expanded
             else:
-                ret_str = self.raw.rstrip()
-
-                # In non-verbose mode, display raw multiline commands on 1 line
-                if self.statement.multiline_command:
-                    # This is an approximation and not meant to be a perfect piecing together of lines.
-                    # All newlines will be converted to spaces, including the ones in quoted strings that
-                    # are considered literals. Also, if the final line starts with a terminator, then the
-                    # terminator will have an extra space before it in the 1 line version.
-                    ret_str = ret_str.replace('\n', ' ')
+                ret_str = single_line_format(self.statement)
 
             # Display a numbered list if not writing to a script
             if not script:

cmd2/rl_utils.py

@@ -38,7 +38,7 @@
 
 
 class RlType(Enum):
-    """Readline library types we recognize"""
+    """Readline library types we support"""
 
     GNU = 1
     PYREADLINE = 2
@@ -151,7 +151,7 @@ def pyreadline_remove_history_item(pos: int) -> None:
             rl_type = RlType.GNU
             vt100_support = sys.stdout.isatty()
 
-# Check if readline was loaded
+# Check if we loaded a supported version of readline
 if rl_type == RlType.NONE:  # pragma: no cover
     if not _rl_warn_reason:
         _rl_warn_reason = (

tests/test_cmd2.py

@@ -6,6 +6,7 @@
 import builtins
 import io
 import os
+import readline
 import signal
 import sys
 import tempfile
@@ -1637,6 +1638,100 @@ def test_multiline_input_line_to_statement(multiline_app):
     assert statement.multiline_command == 'orate'
 
 
+def test_multiline_history_no_prior_history(multiline_app):
+    # Test no existing history prior to typing the command
+    m = mock.MagicMock(name='input', side_effect=['person', '\n'])
+    builtins.input = m
+
+    # Set orig_rl_history_length to 0 before the first line is typed.
+    readline.clear_history()
+    orig_rl_history_length = readline.get_current_history_length()
+
+    line = "orate hi"
+    readline.add_history(line)
+    multiline_app._input_line_to_statement(line, orig_rl_history_length=orig_rl_history_length)
+
+    assert readline.get_current_history_length() == orig_rl_history_length + 1
+    assert readline.get_history_item(1) == "orate hi person"
+
+
+def test_multiline_history_first_line_matches_prev_entry(multiline_app):
+    # Test when first line of multiline command matches previous history entry
+    m = mock.MagicMock(name='input', side_effect=['person', '\n'])
+    builtins.input = m
+
+    # Since the first line of our command matches the previous entry,
+    # orig_rl_history_length is set before the first line is typed.
+    line = "orate hi"
+    readline.clear_history()
+    readline.add_history(line)
+    orig_rl_history_length = readline.get_current_history_length()
+
+    multiline_app._input_line_to_statement(line, orig_rl_history_length=orig_rl_history_length)
+
+    assert readline.get_current_history_length() == orig_rl_history_length + 1
+    assert readline.get_history_item(1) == line
+    assert readline.get_history_item(2) == "orate hi person"
+
+
+def test_multiline_history_matches_prev_entry(multiline_app):
+    # Test combined multiline command that matches previous history entry
+    m = mock.MagicMock(name='input', side_effect=['person', '\n'])
+    builtins.input = m
+
+    readline.clear_history()
+    readline.add_history("orate hi person")
+    orig_rl_history_length = readline.get_current_history_length()
+
+    line = "orate hi"
+    readline.add_history(line)
+    multiline_app._input_line_to_statement(line, orig_rl_history_length=orig_rl_history_length)
+
+    # Since it matches the previous history item, nothing was added to readline history
+    assert readline.get_current_history_length() == orig_rl_history_length
+    assert readline.get_history_item(1) == "orate hi person"
+
+
+def test_multiline_history_does_not_match_prev_entry(multiline_app):
+    # Test combined multiline command that does not match previous history entry
+    m = mock.MagicMock(name='input', side_effect=['person', '\n'])
+    builtins.input = m
+
+    readline.clear_history()
+    readline.add_history("no match")
+    orig_rl_history_length = readline.get_current_history_length()
+
+    line = "orate hi"
+    readline.add_history(line)
+    multiline_app._input_line_to_statement(line, orig_rl_history_length=orig_rl_history_length)
+
+    # Since it doesn't match the previous history item, it was added to readline history
+    assert readline.get_current_history_length() == orig_rl_history_length + 1
+    assert readline.get_history_item(1) == "no match"
+    assert readline.get_history_item(2) == "orate hi person"
+
+
+def test_multiline_history_with_quotes(multiline_app):
+    # Test combined multiline command with quotes
+    m = mock.MagicMock(name='input', side_effect=['  and spaces  ', ' "', ' in', 'quotes.', ';'])
+    builtins.input = m
+
+    readline.clear_history()
+    orig_rl_history_length = readline.get_current_history_length()
+
+    line = 'orate Look, "There are newlines'
+    readline.add_history(line)
+    multiline_app._input_line_to_statement(line, orig_rl_history_length=orig_rl_history_length)
+
+    # Since spaces and newlines in quotes are preserved, this history entry spans multiple lines.
+    assert readline.get_current_history_length() == orig_rl_history_length + 1
+
+    history_lines = readline.get_history_item(1).splitlines()
+    assert history_lines[0] == 'orate Look, "There are newlines'
+    assert history_lines[1] == '  and spaces  '
+    assert history_lines[2] == ' " in quotes.;'
+
+
 class CommandResultApp(cmd2.Cmd):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1731,8 +1826,6 @@ def test_read_input_rawinput_true(capsys, monkeypatch):
         assert line == input_str
 
         # Run custom history code
-        import readline
-
         readline.add_history('old_history')
         custom_history = ['cmd1', 'cmd2']
         line = app.read_input(prompt_str, history=custom_history, completion_mode=cmd2.CompletionMode.NONE)