Merged from master and resolved conflicts

Also:
- Fixed a couple warnings which showed up in VScode.

Author: tleonhardt

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.8.8 (TBD, 2018)
+* Bug Fixes
+    * Prevent crashes that could occur attempting to open a file in non-existent directory or with very long filename
+    
 ## 0.9.1 (May 28, 2018)
 * Bug Fixes
     * fix packaging error for 0.8.x versions (yes we had to deploy a new version

cmd2/argparse_completer.py

@@ -59,6 +59,7 @@ def my_completer(text: str, line: str, begidx: int, endidx:int, extra_param: str
 
 import argparse
 from colorama import Fore
+import os
 import sys
 from typing import List, Dict, Tuple, Callable, Union
 
@@ -78,6 +79,15 @@ def my_completer(text: str, line: str, begidx: int, endidx:int, extra_param: str
 ACTION_SUPPRESS_HINT = 'suppress_hint'
 
 
+class CompletionItem(str):
+    def __new__(cls, o, desc='', *args, **kwargs):
+        return str.__new__(cls, o, *args, **kwargs)
+
+    # noinspection PyMissingConstructor,PyUnusedLocal
+    def __init__(self, o, desc='', *args, **kwargs):
+        self.description = desc
+
+
 class _RangeAction(object):
     def __init__(self, nargs: Union[int, str, Tuple[int, int], None]):
         self.nargs_min = None
@@ -413,6 +423,8 @@ def consume_positional_argument() -> None:
                 completion_results = self._complete_for_arg(flag_action, text, line, begidx, endidx, consumed)
                 if not completion_results:
                     self._print_action_help(flag_action)
+                elif len(completion_results) > 1:
+                    completion_results = self._format_completions(flag_action, completion_results)
 
         # ok, we're not a flag, see if there's a positional argument to complete
         else:
@@ -422,9 +434,35 @@ def consume_positional_argument() -> None:
                 completion_results = self._complete_for_arg(pos_action, text, line, begidx, endidx, consumed)
                 if not completion_results:
                     self._print_action_help(pos_action)
+                elif len(completion_results) > 1:
+                    completion_results = self._format_completions(pos_action, completion_results)
 
         return completion_results
 
+    def _format_completions(self, action, completions: List[Union[str, CompletionItem]]):
+        if completions and len(completions) > 1 and isinstance(completions[0], CompletionItem):
+            token_width = len(action.dest)
+            completions_with_desc = []
+
+            for item in completions:
+                if len(item) > token_width:
+                    token_width = len(item)
+
+            term_size = os.get_terminal_size()
+            fill_width = int(term_size.columns * .6) - (token_width + 2)
+            for item in completions:
+                entry = '{: <{token_width}}{: <{fill_width}}'.format(item, item.description,
+                                                                     token_width=token_width+2,
+                                                                     fill_width=fill_width)
+                completions_with_desc.append(entry)
+
+            header = '\n{: <{token_width}}{}'.format(action.dest.upper(), action.desc_header, token_width=token_width+2)
+
+            self._cmd2_app.completion_header = header
+            self._cmd2_app.display_matches = completions_with_desc
+
+        return completions
+
     def complete_command_help(self, tokens: List[str], text: str, line: str, begidx: int, endidx: int) -> List[str]:
         """Supports the completion of sub-commands for commands through the cmd2 help command."""
         for idx, token in enumerate(tokens):

cmd2/cmd2.py

@@ -494,13 +494,20 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         # will be added if there is an unmatched opening quote
         self.allow_closing_quote = True
 
+        # An optional header that prints above the tab-completion suggestions
+        self.completion_header = ''
+
         # Use this list if you are completing strings that contain a common delimiter and you only want to
         # display the final portion of the matches as the tab-completion suggestions. The full matches
         # still must be returned from your completer function. For an example, look at path_complete()
         # which uses this to show only the basename of paths as the suggestions. delimiter_complete() also
         # populates this list.
         self.display_matches = []
 
+        # Used by functions like path_complete() and delimiter_complete() to properly
+        # quote matches that are completed in a delimited fashion
+        self.matches_delimited = False
+
     # -----  Methods related to presenting output to the user -----
 
     @property
@@ -657,7 +664,9 @@ def reset_completion_defaults(self):
         """
         self.allow_appended_space = True
         self.allow_closing_quote = True
+        self.completion_header = ''
         self.display_matches = []
+        self.matches_delimited = False
 
         if rl_type == RlType.GNU:
             readline.set_completion_display_matches_hook(self._display_matches_gnu_readline)
@@ -836,6 +845,8 @@ def delimiter_complete(self, text, line, begidx, endidx, match_against, delimite
 
         # Display only the portion of the match that's being completed based on delimiter
         if matches:
+            # Set this to True for proper quoting of matches with spaces
+            self.matches_delimited = True
 
             # Get the common beginning for the matches
             common_prefix = os.path.commonprefix(matches)
@@ -1037,6 +1048,9 @@ def complete_users():
                 search_str = os.path.join(os.getcwd(), search_str)
                 cwd_added = True
 
+        # Set this to True for proper quoting of paths with spaces
+        self.matches_delimited = True
+
         # Find all matching path completions
         matches = glob.glob(search_str)
 
@@ -1245,6 +1259,10 @@ def _display_matches_gnu_readline(self, substitution, matches, longest_match_len
             strings_array[1:-1] = encoded_matches
             strings_array[-1] = None
 
+            # Print the header if one exists
+            if self.completion_header:
+                sys.stdout.write('\n' + self.completion_header)
+
             # Call readline's display function
             # rl_display_match_list(strings_array, number of completion matches, longest match length)
             readline_lib.rl_display_match_list(strings_array, len(encoded_matches), longest_match_length)
@@ -1270,6 +1288,10 @@ def _display_matches_pyreadline(self, matches):  # pragma: no cover
             # Add padding for visual appeal
             matches_to_display, _ = self._pad_matches_to_display(matches_to_display)
 
+            # Print the header if one exists
+            if self.completion_header:
+                readline.rl.mode.console.write('\n' + self.completion_header)
+
             # Display matches using actual display function. This also redraws the prompt and line.
             orig_pyreadline_display(matches_to_display)
 
@@ -1309,7 +1331,7 @@ def complete(self, text, state):
             # from text and update the indexes. This only applies if we are at the the beginning of the line.
             shortcut_to_restore = ''
             if begidx == 0:
-                for (shortcut, expansion) in self.shortcuts:
+                for (shortcut, _) in self.shortcuts:
                     if text.startswith(shortcut):
                         # Save the shortcut to restore later
                         shortcut_to_restore = shortcut
@@ -1414,13 +1436,7 @@ def complete(self, text, state):
                     display_matches_set = set(self.display_matches)
                     self.display_matches = list(display_matches_set)
 
-                    # Check if display_matches has been used. If so, then matches
-                    # on delimited strings like paths was done.
-                    if self.display_matches:
-                        matches_delimited = True
-                    else:
-                        matches_delimited = False
-
+                    if not self.display_matches:
                         # Since self.display_matches is empty, set it to self.completion_matches
                         # before we alter them. That way the suggestions will reflect how we parsed
                         # the token being completed and not how readline did.
@@ -1435,7 +1451,7 @@ def complete(self, text, state):
                         # This is the tab completion text that will appear on the command line.
                         common_prefix = os.path.commonprefix(self.completion_matches)
 
-                        if matches_delimited:
+                        if self.matches_delimited:
                             # Check if any portion of the display matches appears in the tab completion
                             display_prefix = os.path.commonprefix(self.display_matches)
 
@@ -1696,7 +1712,7 @@ def onecmd_plus_hooks(self, line):
                 if self.timing:
                     self.pfeedback('Elapsed: %s' % str(datetime.datetime.now() - timestart))
             finally:
-                if self.allow_redirection:
+                if self.allow_redirection and self.redirecting:
                     self._restore_output(statement)
         except EmptyStatement:
             pass
@@ -1840,7 +1856,11 @@ def _redirect_output(self, statement):
                 # REDIRECTION_APPEND or REDIRECTION_OUTPUT
                 if statement.output == constants.REDIRECTION_APPEND:
                     mode = 'a'
-                sys.stdout = self.stdout = open(statement.output_to, mode)
+                try:
+                    sys.stdout = self.stdout = open(statement.output_to, mode)
+                except OSError as ex:
+                    self.perror('Not Redirecting because - {}'.format(ex), traceback_war=False)
+                    self.redirecting = False
             else:
                 # going to a paste buffer
                 sys.stdout = self.stdout = tempfile.TemporaryFile(mode="w+")
@@ -2366,7 +2386,7 @@ def select(self, opts, prompt='Your choice? '):
                     fulloptions.append((opt[0], opt[1]))
                 except IndexError:
                     fulloptions.append((opt[0], opt[0]))
-        for (idx, (value, text)) in enumerate(fulloptions):
+        for (idx, (_, text)) in enumerate(fulloptions):
             self.poutput('  %2d. %s\n' % (idx + 1, text))
         while True:
             response = input(prompt)
@@ -2878,16 +2898,19 @@ def _generate_transcript(self, history, transcript_file):
         self.echo = saved_echo
 
         # finally, we can write the transcript out to the file
-        with open(transcript_file, 'w') as fout:
-            fout.write(transcript)
-
-        # and let the user know what we did
-        if len(history) > 1:
-            plural = 'commands and their outputs'
+        try:
+            with open(transcript_file, 'w') as fout:
+                fout.write(transcript)
+        except OSError as ex:
+            self.perror('Failed to save transcript: {}'.format(ex), traceback_war=False)
         else:
-            plural = 'command and its output'
-        msg = '{} {} saved to transcript file {!r}'
-        self.pfeedback(msg.format(len(history), plural, transcript_file))
+            # and let the user know what we did
+            if len(history) > 1:
+                plural = 'commands and their outputs'
+            else:
+                plural = 'command and its output'
+            msg = '{} {} saved to transcript file {!r}'
+            self.pfeedback(msg.format(len(history), plural, transcript_file))
 
     @with_argument_list
     def do_edit(self, arglist):

examples/tab_autocompletion.py

@@ -109,6 +109,18 @@ def instance_query_actors(self) -> List[str]:
         """Simulating a function that queries and returns a completion values"""
         return actors
 
+    def instance_query_movie_ids(self) -> List[str]:
+        """Demonstrates showing tabular hinting of tab completion information"""
+        completions_with_desc = []
+
+        for movie_id, movie_entry in self.MOVIE_DATABASE.items():
+            completions_with_desc.append(argparse_completer.CompletionItem(movie_id, movie_entry['title']))
+
+        setattr(self.vid_delete_movie_id, 'desc_header', 'Title')
+        setattr(self.movies_delete_movie_id, 'desc_header', 'Title')
+
+        return completions_with_desc
+
     # This demonstrates a number of customizations of the AutoCompleter version of ArgumentParser
     #  - The help output will separately group required vs optional flags
     #  - The help output for arguments with multiple flags or with append=True is more concise
@@ -253,6 +265,8 @@ def _do_vid_media_shows(self, args) -> None:
             ('path_complete', [False, False]))
 
     vid_movies_delete_parser = vid_movies_commands_subparsers.add_parser('delete')
+    vid_delete_movie_id = vid_movies_delete_parser.add_argument('movie_id', help='Movie ID')
+    setattr(vid_delete_movie_id, argparse_completer.ACTION_ARG_CHOICES, instance_query_movie_ids)
 
     vid_shows_parser = video_types_subparsers.add_parser('shows')
     vid_shows_parser.set_defaults(func=_do_vid_media_shows)
@@ -328,6 +342,8 @@ def _do_media_shows(self, args) -> None:
     movies_add_parser.add_argument('actor', help='Actors', nargs='*')
 
     movies_delete_parser = movies_commands_subparsers.add_parser('delete')
+    movies_delete_movie_id = movies_delete_parser.add_argument('movie_id', help='Movie ID')
+    setattr(movies_delete_movie_id, argparse_completer.ACTION_ARG_CHOICES, 'instance_query_movie_ids')
 
     movies_load_parser = movies_commands_subparsers.add_parser('load')
     movie_file_action = movies_load_parser.add_argument('movie_file', help='Movie database')
@@ -362,7 +378,7 @@ def complete_media(self, text, line, begidx, endidx):
                    'director': TabCompleteExample.static_list_directors,  # static list
                    'movie_file': (self.path_complete, [False, False])
                    }
-        completer = argparse_completer.AutoCompleter(TabCompleteExample.media_parser, arg_choices=choices)
+        completer = argparse_completer.AutoCompleter(TabCompleteExample.media_parser, arg_choices=choices, cmd2_app=self)
 
         tokens, _ = self.tokens_for_completion(line, begidx, endidx)
         results = completer.complete_command(tokens, text, line, begidx, endidx)

tests/test_cmd2.py

@@ -615,6 +615,44 @@ def test_output_redirection(base_app):
     finally:
         os.remove(filename)
 
+def test_output_redirection_to_nonexistent_directory(base_app):
+    filename = '~/fakedir/this_does_not_exist.txt'
+
+    # Verify that writing to a file in a non-existent directory doesn't work
+    run_cmd(base_app, 'help > {}'.format(filename))
+    expected = normalize(BASE_HELP)
+    with pytest.raises(FileNotFoundError):
+        with open(filename) as f:
+            content = normalize(f.read())
+        assert content == expected
+
+    # Verify that appending to a file also works
+    run_cmd(base_app, 'help history >> {}'.format(filename))
+    expected = normalize(BASE_HELP + '\n' + HELP_HISTORY)
+    with pytest.raises(FileNotFoundError):
+        with open(filename) as f:
+            content = normalize(f.read())
+        assert content == expected
+
+def test_output_redirection_to_too_long_filename(base_app):
+    filename = '~/sdkfhksdjfhkjdshfkjsdhfkjsdhfkjdshfkjdshfkjshdfkhdsfkjhewfuihewiufhweiufhiweufhiuewhiuewhfiuwehfiuewhfiuewhfiuewhfiuewhiuewhfiuewhfiuewfhiuwehewiufhewiuhfiweuhfiuwehfiuewfhiuwehiuewfhiuewhiewuhfiuewhfiuwefhewiuhewiufhewiufhewiufhewiufhewiufhewiufhewiufhewiuhewiufhewiufhewiuheiufhiuewheiwufhewiufheiufheiufhieuwhfewiuhfeiufhiuewfhiuewheiwuhfiuewhfiuewhfeiuwfhewiufhiuewhiuewhfeiuwhfiuwehfuiwehfiuehiuewhfieuwfhieufhiuewhfeiuwfhiuefhueiwhfw'
+
+    # Verify that writing to a file in a non-existent directory doesn't work
+    run_cmd(base_app, 'help > {}'.format(filename))
+    expected = normalize(BASE_HELP)
+    with pytest.raises(OSError):
+        with open(filename) as f:
+            content = normalize(f.read())
+        assert content == expected
+
+    # Verify that appending to a file also works
+    run_cmd(base_app, 'help history >> {}'.format(filename))
+    expected = normalize(BASE_HELP + '\n' + HELP_HISTORY)
+    with pytest.raises(OSError):
+        with open(filename) as f:
+            content = normalize(f.read())
+        assert content == expected
+
 
 def test_feedback_to_output_true(base_app):
     base_app.feedback_to_output = True

tests/test_transcript.py

@@ -154,6 +154,32 @@ def test_history_transcript(request, capsys):
 
     assert transcript == expected
 
+def test_history_transcript_bad_filename(request, capsys):
+    app = CmdLineApp()
+    app.stdout = StdOut()
+    run_cmd(app, 'orate this is\na /multiline/\ncommand;\n')
+    run_cmd(app, 'speak /tmp/file.txt is not a regex')
+
+    expected = r"""(Cmd) orate this is
+> a /multiline/
+> command;
+this is a \/multiline\/ command
+(Cmd) speak /tmp/file.txt is not a regex
+\/tmp\/file.txt is not a regex
+"""
+
+    # make a tmp file
+    history_fname = '~/fakedir/this_does_not_exist.txt'
+
+    # tell the history command to create a transcript
+    run_cmd(app, 'history -t "{}"'.format(history_fname))
+
+    # read in the transcript created by the history command
+    with pytest.raises(FileNotFoundError):
+        with open(history_fname) as f:
+            transcript = f.read()
+        assert transcript == expected
+
 @pytest.mark.parametrize('expected, transformed', [
     # strings with zero or one slash or with escaped slashes means no regular
     # expression present, so the result should just be what re.escape returns.