Merged master into delete_optparse branch

Also:
- Bumped version to 0.9.0

1 files changed, 160 insertions, 207 deletions

diff --git a/cmd2.py b/cmd2.py
index 5d15f884..057f95f1 100755
--- a/cmd2.py
+++ b/cmd2.py
@@ -111,7 +111,7 @@ if sys.version_info < (3, 5):
 else:
     from contextlib import redirect_stdout, redirect_stderr
 
-if sys.version_info > (3, 0):
+if six.PY3:
     from io import StringIO  # Python3
 else:
     from io import BytesIO as StringIO  # Python2
@@ -186,7 +186,7 @@ if six.PY2 and sys.platform.startswith('lin'):
     except ImportError:
         pass
 
-__version__ = '0.8.5'
+__version__ = '0.9.0'
 
 # Pyparsing enablePackrat() can greatly speed up parsing, but problems have been seen in Python 3 in the past
 pyparsing.ParserElement.enablePackrat()
@@ -305,7 +305,7 @@ def with_argument_list(func):
     @functools.wraps(func)
     def cmd_wrapper(self, cmdline):
         lexed_arglist = parse_quoted_string(cmdline)
-        func(self, lexed_arglist)
+        return func(self, lexed_arglist)
 
     cmd_wrapper.__doc__ = func.__doc__
     return cmd_wrapper
@@ -325,7 +325,7 @@ def with_argparser_and_unknown_args(argparser):
         def cmd_wrapper(instance, cmdline):
             lexed_arglist = parse_quoted_string(cmdline)
             args, unknown = argparser.parse_known_args(lexed_arglist)
-            func(instance, args, unknown)
+            return func(instance, args, unknown)
 
         # argparser defaults the program name to sys.argv[0]
         # we want it to be the name of our command
@@ -367,7 +367,7 @@ def with_argparser(argparser):
         def cmd_wrapper(instance, cmdline):
             lexed_arglist = parse_quoted_string(cmdline)
             args = argparser.parse_args(lexed_arglist)
-            func(instance, args)
+            return func(instance, args)
 
         # argparser defaults the program name to sys.argv[0]
         # we want it to be the name of our command
@@ -931,8 +931,11 @@ class Cmd(cmd.Cmd):
         # will be added if there is an unmatched opening quote
         self.allow_closing_quote = True
 
-        # If the tab-completion matches should be displayed in a way that is different than the actual match values,
-        # then place those results in this list. path_complete uses this to show only the basename of completions.
+        # Use this list if you are completing strings that contain a common delimiter and you only want to
+        # display the final portion of the matches as the tab-completion suggestions. The full matches
+        # still must be returned from your completer function. For an example, look at path_complete()
+        # which uses this to show only the basename of paths as the suggestions. delimiter_complete() also
+        # populates this list.
         self.display_matches = []
 
     # -----  Methods related to presenting output to the user -----
@@ -1145,7 +1148,7 @@ class Cmd(cmd.Cmd):
                  On Success
                      tokens: list of unquoted tokens
                              this is generally the list needed for tab completion functions
-                     raw_tokens: list of tokens as they appear on the command line, meaning their quotes are preserved
+                     raw_tokens: list of tokens with any quotes preserved
                                  this can be used to know if a token was quoted or is missing a closing quote
 
                      Both lists are guaranteed to have at least 1 item
@@ -1173,7 +1176,7 @@ class Cmd(cmd.Cmd):
                 break
             except ValueError:
                 # ValueError can be caused by missing closing quote
-                if len(quotes_to_try) == 0:
+                if not quotes_to_try:
                     # Since we have no more quotes to try, something else
                     # is causing the parsing error. Return None since
                     # this means the line is malformed.
@@ -1305,7 +1308,7 @@ class Cmd(cmd.Cmd):
         matches = self.basic_complete(text, line, begidx, endidx, match_against)
 
         # Display only the portion of the match that's being completed based on delimiter
-        if len(matches) > 0:
+        if matches:
 
             # Get the common beginning for the matches
             common_prefix = os.path.commonprefix(matches)
@@ -1313,7 +1316,7 @@ class Cmd(cmd.Cmd):
 
             # Calculate what portion of the match we are completing
             display_token_index = 0
-            if len(prefix_tokens) > 0:
+            if prefix_tokens:
                 display_token_index = len(prefix_tokens) - 1
 
             # Get this portion for each match and store them in self.display_matches
@@ -1321,7 +1324,7 @@ class Cmd(cmd.Cmd):
                 match_tokens = cur_match.split(delimiter)
                 display_token = match_tokens[display_token_index]
 
-                if len(display_token) == 0:
+                if not display_token:
                     display_token = delimiter
                 self.display_matches.append(display_token)
 
@@ -1423,6 +1426,42 @@ class Cmd(cmd.Cmd):
         :param dir_only: bool - only return directories
         :return: List[str] - a list of possible tab completions
         """
+
+        # Used to complete ~ and ~user strings
+        def complete_users():
+
+            # We are returning ~user strings that resolve to directories,
+            # so don't append a space or quote in the case of a single result.
+            self.allow_appended_space = False
+            self.allow_closing_quote = False
+
+            users = []
+
+            # Windows lacks the pwd module so we can't get a list of users.
+            # Instead we will add a slash once the user enters text that
+            # resolves to an existing home directory.
+            if sys.platform.startswith('win'):
+                expanded_path = os.path.expanduser(text)
+                if os.path.isdir(expanded_path):
+                    users.append(text + os.path.sep)
+            else:
+                import pwd
+
+                # Iterate through a list of users from the password database
+                for cur_pw in pwd.getpwall():
+
+                    # Check if the user has an existing home dir
+                    if os.path.isdir(cur_pw.pw_dir):
+
+                        # Add a ~ to the user to match against text
+                        cur_user = '~' + cur_pw.pw_name
+                        if cur_user.startswith(text):
+                            if add_trailing_sep_if_dir:
+                                cur_user += os.path.sep
+                            users.append(cur_user)
+
+            return users
+
         # Determine if a trailing separator should be appended to directory completions
         add_trailing_sep_if_dir = False
         if endidx == len(line) or (endidx < len(line) and line[endidx] != os.path.sep):
@@ -1432,9 +1471,9 @@ class Cmd(cmd.Cmd):
         cwd = os.getcwd()
         cwd_added = False
 
-        # Used to replace ~ in the final results
-        user_path = os.path.expanduser('~')
-        tilde_expanded = False
+        # Used to replace expanded user path in final result
+        orig_tilde_path = ''
+        expanded_tilde_path = ''
 
         # If the search text is blank, then search in the CWD for *
         if not text:
@@ -1447,35 +1486,30 @@ class Cmd(cmd.Cmd):
                 if wildcard in text:
                     return []
 
-            # Used if we need to prepend a directory to the search string
-            dirname = ''
+            # Start the search string
+            search_str = text + '*'
 
-            # If the user only entered a '~', then complete it with a slash
-            if text == '~':
-                # This is a directory, so don't add a space or quote
-                self.allow_appended_space = False
-                self.allow_closing_quote = False
-                return [text + os.path.sep]
+            # Handle tilde expansion and completion
+            if text.startswith('~'):
+                sep_index = text.find(os.path.sep, 1)
 
-            elif text.startswith('~'):
-                # Tilde without separator between path is invalid
-                if not text.startswith('~' + os.path.sep):
-                    return []
+                # If there is no slash, then the user is still completing the user after the tilde
+                if sep_index == -1:
+                    return complete_users()
+
+                # Otherwise expand the user dir
+                else:
+                    search_str = os.path.expanduser(search_str)
 
-                # Mark that we are expanding a tilde
-                tilde_expanded = True
+                    # Get what we need to restore the original tilde path later
+                    orig_tilde_path = text[:sep_index]
+                    expanded_tilde_path = os.path.expanduser(orig_tilde_path)
 
             # If the search text does not have a directory, then use the cwd
             elif not os.path.dirname(text):
-                dirname = os.getcwd()
+                search_str = os.path.join(os.getcwd(), search_str)
                 cwd_added = True
 
-            # Build the search string
-            search_str = os.path.join(dirname, text + '*')
-
-            # Expand "~" to the real user directory
-            search_str = os.path.expanduser(search_str)
-
         # Find all matching path completions
         matches = glob.glob(search_str)
 
@@ -1486,7 +1520,7 @@ class Cmd(cmd.Cmd):
             matches = [c for c in matches if os.path.isdir(c)]
 
         # Don't append a space or closing quote to directory
-        if len(matches) == 1 and not os.path.isfile(matches[0]):
+        if len(matches) == 1 and os.path.isdir(matches[0]):
             self.allow_appended_space = False
             self.allow_closing_quote = False
 
@@ -1501,13 +1535,13 @@ class Cmd(cmd.Cmd):
                 matches[index] += os.path.sep
                 self.display_matches[index] += os.path.sep
 
-        # Remove cwd if it was added
+        # Remove cwd if it was added to match the text readline expects
         if cwd_added:
             matches = [cur_path.replace(cwd + os.path.sep, '', 1) for cur_path in matches]
 
-        # Restore a tilde if we expanded one
-        if tilde_expanded:
-            matches = [cur_path.replace(user_path, '~', 1) for cur_path in matches]
+        # Restore the tilde string if we expanded one to match the text readline expects
+        if expanded_tilde_path:
+            matches = [cur_path.replace(expanded_tilde_path, orig_tilde_path, 1) for cur_path in matches]
 
         return matches
 
@@ -1552,11 +1586,11 @@ class Cmd(cmd.Cmd):
         :return: List[str] - a list of possible tab completions
         """
         # Don't tab complete anything if no shell command has been started
-        if not complete_blank and len(text) == 0:
+        if not complete_blank and not text:
             return []
 
         # If there are no path characters in the search text, then do shell command completion in the user's path
-        if os.path.sep not in text:
+        if not text.startswith('~') and os.path.sep not in text:
             return self.get_exes_in_path(text)
 
         # Otherwise look for executables in the given path
@@ -1630,9 +1664,6 @@ class Cmd(cmd.Cmd):
         :param matches_to_display: the matches being padded
         :return: the padded matches and length of padding that was added
         """
-        if rl_type == RlType.NONE:
-            return matches_to_display, 0
-
         if rl_type == RlType.GNU:
             # Add 2 to the padding of 2 that readline uses for a total of 4.
             padding = 2 * ' '
@@ -1641,6 +1672,9 @@ class Cmd(cmd.Cmd):
             # Add 3 to the padding of 1 that pyreadline uses for a total of 4.
             padding = 3 * ' '
 
+        else:
+            return matches_to_display, 0
+
         return [cur_match + padding for cur_match in matches_to_display], len(padding)
 
     def _display_matches_gnu_readline(self, substitution, matches, longest_match_length):
@@ -1655,7 +1689,7 @@ class Cmd(cmd.Cmd):
         if rl_type == RlType.GNU:
 
             # Check if we should show display_matches
-            if len(self.display_matches) > 0:
+            if self.display_matches:
                 matches_to_display = self.display_matches
 
                 # Recalculate longest_match_length for display_matches
@@ -1713,7 +1747,7 @@ class Cmd(cmd.Cmd):
         if rl_type == RlType.PYREADLINE:
 
             # Check if we should show display_matches
-            if len(self.display_matches) > 0:
+            if self.display_matches:
                 matches_to_display = self.display_matches
             else:
                 matches_to_display = matches
@@ -1724,117 +1758,6 @@ class Cmd(cmd.Cmd):
             # Display the matches
             orig_pyreadline_display(matches_to_display)
 
-    def _handle_completion_token_quote(self, raw_completion_token):
-        """
-        This is called by complete() to add an opening quote to the token being completed if it is needed
-        The readline input buffer is then updated with the new string
-        :param raw_completion_token: str - the token being completed as it appears on the command line
-        :return: True if a quote was added, False otherwise
-        """
-        if len(self.completion_matches) == 0:
-            return False
-
-        quote_added = False
-
-        # Check if token on screen is already quoted
-        if len(raw_completion_token) == 0 or raw_completion_token[0] not in QUOTES:
-
-            # Get the common prefix of all matches. This is what be written to the screen.
-            common_prefix = os.path.commonprefix(self.completion_matches)
-
-            # If common_prefix contains a space, then we must add an opening quote to it
-            if ' ' in common_prefix:
-
-                # Figure out what kind of quote to add
-                if '"' in common_prefix:
-                    quote = "'"
-                else:
-                    quote = '"'
-
-                new_completion_token = quote + common_prefix
-
-                # Handle a single result
-                if len(self.completion_matches) == 1:
-                    str_to_append = ''
-
-                    # Add a closing quote if allowed
-                    if self.allow_closing_quote:
-                        str_to_append += quote
-
-                    orig_line = readline.get_line_buffer()
-                    endidx = readline.get_endidx()
-
-                    # If we are at the end of the line, then add a space if allowed
-                    if self.allow_appended_space and endidx == len(orig_line):
-                        str_to_append += ' '
-
-                    new_completion_token += str_to_append
-
-                # Update the line
-                quote_added = True
-                self._replace_completion_token(raw_completion_token, new_completion_token)
-
-        return quote_added
-
-    def _replace_completion_token(self, raw_completion_token, new_completion_token):
-        """
-        Replaces the token being completed in the readline line buffer which updates the screen
-        This is used for things like adding an opening quote for completions with spaces
-        :param raw_completion_token: str - the original token being completed as it appears on the command line
-        :param new_completion_token: str- the replacement token
-        :return: None
-        """
-        orig_line = readline.get_line_buffer()
-        endidx = readline.get_endidx()
-
-        starting_index = orig_line[:endidx].rfind(raw_completion_token)
-
-        if starting_index != -1:
-            # Build the new line
-            new_line = orig_line[:starting_index]
-            new_line += new_completion_token
-            new_line += orig_line[endidx:]
-
-            # Calculate the new cursor offset
-            len_diff = len(new_completion_token) - len(raw_completion_token)
-            new_point = endidx + len_diff
-
-            # Replace the line and update the cursor offset
-            self._set_readline_line(new_line)
-            self._set_readline_point(new_point)
-
-    @staticmethod
-    def _set_readline_line(new_line):
-        """
-        Sets the readline line buffer
-        :param new_line: str - the new line contents
-        """
-        if rl_type == RlType.GNU:
-            # Byte encode the new line
-            if six.PY3:
-                encoded_line = bytes(new_line, encoding='utf-8')
-            else:
-                encoded_line = bytes(new_line)
-
-            # Replace the line
-            readline_lib.rl_replace_line(encoded_line, 0)
-
-        elif rl_type == RlType.PYREADLINE:
-            readline.rl.mode.l_buffer.set_line(new_line)
-
-    @staticmethod
-    def _set_readline_point(new_point):
-        """
-        Sets the cursor offset in the readline line buffer
-        :param new_point: int - the new cursor offset
-        """
-        if rl_type == RlType.GNU:
-            rl_point = ctypes.c_int.in_dll(readline_lib, "rl_point")
-            rl_point.value = new_point
-
-        elif rl_type == RlType.PYREADLINE:
-            readline.rl.mode.l_buffer.point = new_point
-
     # -----  Methods which override stuff in cmd -----
 
     def complete(self, text, state):
@@ -1865,10 +1788,9 @@ class Cmd(cmd.Cmd):
             begidx = max(readline.get_begidx() - stripped, 0)
             endidx = max(readline.get_endidx() - stripped, 0)
 
-            # We only break words on whitespace and quotes when tab completing.
-            # Therefore shortcuts become part of the text variable if there isn't a space after it.
-            # We need to remove it from text and update the indexes. This only applies if we are at
-            # the beginning of the line.
+            # Shortcuts are not word break characters when tab completing. Therefore shortcuts become part
+            # of the text variable if there isn't a word break, like a space, after it. We need to remove it
+            # from text and update the indexes. This only applies if we are at the the beginning of the line.
             shortcut_to_restore = ''
             if begidx == 0:
                 for (shortcut, expansion) in self.shortcuts:
@@ -1912,21 +1834,32 @@ class Cmd(cmd.Cmd):
                     self.completion_matches = []
                     return None
 
-                # readline still performs word breaks after a quote. Therefore something like quoted search
-                # text with a space would have resulted in begidx pointing to the middle of the token we
-                # we want to complete. Figure out where that token actually begins and save the beginning
-                # portion of it that was not part of the text readline gave us. We will remove it from the
-                # completions later since readline expects them to start with the original text.
-                actual_begidx = line[:endidx].rfind(tokens[-1])
+                # Text we need to remove from completions later
                 text_to_remove = ''
 
-                if actual_begidx != begidx:
-                    text_to_remove = line[actual_begidx:begidx]
+                # Get the token being completed with any opening quote preserved
+                raw_completion_token = raw_tokens[-1]
+
+                # Check if the token being completed has an opening quote
+                if raw_completion_token and raw_completion_token[0] in QUOTES:
 
-                    # Adjust text and where it begins so the completer routines
-                    # get unbroken search text to complete on.
-                    text = text_to_remove + text
-                    begidx = actual_begidx
+                    # Since the token is still being completed, we know the opening quote is unclosed
+                    unclosed_quote = raw_completion_token[0]
+
+                    # readline still performs word breaks after a quote. Therefore something like quoted search
+                    # text with a space would have resulted in begidx pointing to the middle of the token we
+                    # we want to complete. Figure out where that token actually begins and save the beginning
+                    # portion of it that was not part of the text readline gave us. We will remove it from the
+                    # completions later since readline expects them to start with the original text.
+                    actual_begidx = line[:endidx].rfind(tokens[-1])
+
+                    if actual_begidx != begidx:
+                        text_to_remove = line[actual_begidx:begidx]
+
+                        # Adjust text and where it begins so the completer routines
+                        # get unbroken search text to complete on.
+                        text = text_to_remove + text
+                        begidx = actual_begidx
 
                 # Check if a valid command was entered
                 if command in self.get_all_commands():
@@ -1957,7 +1890,7 @@ class Cmd(cmd.Cmd):
                 # call the completer function for the current command
                 self.completion_matches = self._redirect_complete(text, line, begidx, endidx, compfunc)
 
-                if len(self.completion_matches) > 0:
+                if self.completion_matches:
 
                     # Eliminate duplicates
                     matches_set = set(self.completion_matches)
@@ -1966,36 +1899,58 @@ class Cmd(cmd.Cmd):
                     display_matches_set = set(self.display_matches)
                     self.display_matches = list(display_matches_set)
 
-                    # Get the token being completed as it appears on the command line
-                    raw_completion_token = raw_tokens[-1]
-
-                    # Add an opening quote if needed
-                    if self._handle_completion_token_quote(raw_completion_token):
-                        # An opening quote was added and the screen was updated. Return no results.
-                        self.completion_matches = []
-                        return None
+                    # Check if display_matches has been used. If so, then matches
+                    # on delimited strings like paths was done.
+                    if self.display_matches:
+                        matches_delimited = True
+                    else:
+                        matches_delimited = False
 
-                    if text_to_remove or shortcut_to_restore:
-                        # If self.display_matches is empty, then set it to self.completion_matches
+                        # Since self.display_matches is empty, set it to self.completion_matches
                         # before we alter them. That way the suggestions will reflect how we parsed
                         # the token being completed and not how readline did.
-                        if len(self.display_matches) == 0:
-                            self.display_matches = copy.copy(self.completion_matches)
+                        self.display_matches = copy.copy(self.completion_matches)
 
-                        # Check if we need to remove text from the beginning of tab completions
-                        if text_to_remove:
-                            self.completion_matches = \
-                                [m.replace(text_to_remove, '', 1) for m in self.completion_matches]
+                    # Check if we need to add an opening quote
+                    if not unclosed_quote:
 
-                        # Check if we need to restore a shortcut in the tab completions
-                        # so it doesn't get erased from the command line
-                        if shortcut_to_restore:
-                            self.completion_matches = \
-                                [shortcut_to_restore + match for match in self.completion_matches]
+                        add_quote = False
 
-                    # If the token being completed starts with a quote then we know it has an unclosed quote
-                    if len(raw_completion_token) > 0 and raw_completion_token[0] in QUOTES:
-                        unclosed_quote = raw_completion_token[0]
+                        # This is the tab completion text that will appear on the command line.
+                        common_prefix = os.path.commonprefix(self.completion_matches)
+
+                        if matches_delimited:
+                            # Check if any portion of the display matches appears in the tab completion
+                            display_prefix = os.path.commonprefix(self.display_matches)
+
+                            # For delimited matches, we check what appears before the display
+                            # matches (common_prefix) as well as the display matches themselves.
+                            if (' ' in common_prefix) or (display_prefix and ' ' in ''.join(self.display_matches)):
+                                add_quote = True
+
+                        # If there is a tab completion and any match has a space, then add an opening quote
+                        elif common_prefix and ' ' in ''.join(self.completion_matches):
+                            add_quote = True
+
+                        if add_quote:
+                            # Figure out what kind of quote to add and save it as the unclosed_quote
+                            if '"' in ''.join(self.completion_matches):
+                                unclosed_quote = "'"
+                            else:
+                                unclosed_quote = '"'
+
+                            self.completion_matches = [unclosed_quote + match for match in self.completion_matches]
+
+                    # Check if we need to remove text from the beginning of tab completions
+                    elif text_to_remove:
+                        self.completion_matches = \
+                            [m.replace(text_to_remove, '', 1) for m in self.completion_matches]
+
+                    # Check if we need to restore a shortcut in the tab completions
+                    # so it doesn't get erased from the command line
+                    if shortcut_to_restore:
+                        self.completion_matches = \
+                            [shortcut_to_restore + match for match in self.completion_matches]
 
             else:
                 # Complete token against aliases and command names
@@ -2019,7 +1974,7 @@ class Cmd(cmd.Cmd):
                 self.completion_matches[0] += str_to_append
 
             # Otherwise sort matches
-            elif len(self.completion_matches) > 0:
+            elif self.completion_matches:
                 self.completion_matches.sort()
                 self.display_matches.sort()
 
@@ -2664,7 +2619,7 @@ Usage:  Usage: alias [name] | [<name> <value>]
         alias save_results "print_results > out.txt"
 """
         # If no args were given, then print a list of current aliases
-        if len(arglist) == 0:
+        if not arglist:
             for cur_alias in self.aliases:
                 self.poutput("alias {} {}".format(cur_alias, self.aliases[cur_alias]))
 
@@ -2712,7 +2667,7 @@ Usage:  Usage: unalias [-a] name [name ...]
     Options:
         -a     remove all alias definitions
 """
-        if len(arglist) == 0:
+        if not arglist:
             self.do_help('unalias')
 
         if '-a' in arglist:
@@ -2826,9 +2781,7 @@ Usage:  Usage: unalias [-a] name [name ...]
                 if self.ruler:
                     self.stdout.write('{:{ruler}<{width}}\n'.format('', ruler=self.ruler, width=80))
 
-                help_topics = self.get_help_topics()
                 for command in cmds:
-                    doc = ''
                     # Try to get the documentation string
                     try:
                         # first see if there's a help function implemented
@@ -2839,7 +2792,7 @@ Usage:  Usage: unalias [-a] name [name ...]
                             # Now see if help_summary has been set
                             doc = getattr(self, self._func_named(command)).help_summary
                         except AttributeError:
-                            # Last, try to directly ac cess the function's doc-string
+                            # Last, try to directly access the function's doc-string
                             doc = getattr(self, self._func_named(command)).__doc__
                     else:
                         # we found the help function
@@ -3026,7 +2979,7 @@ Usage:  Usage: unalias [-a] name [name ...]
 
         # Support expanding ~ in quoted paths
         for index, _ in enumerate(tokens):
-            if len(tokens[index]) > 0:
+            if tokens[index]:
                 # Check if the token is quoted. Since shlex.split() passed, there isn't
                 # an unclosed quote, so we only need to check the first character.
                 first_char = tokens[index][0]