Merge branch 'master' into ignore_identchars

# Conflicts:
#	cmd2/parsing.py
#	tests/test_parsing.py

1 files changed, 39 insertions, 12 deletions

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index eff29843..a1e21175 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -81,7 +81,7 @@ class Statement(str):
         return rtn
 
 
-class StatementParser():
+class StatementParser:
     """Parse raw text into command components.
 
     Shortcuts is a list of tuples with each tuple containing the shortcut and the expansion.
@@ -93,7 +93,7 @@ class StatementParser():
             multiline_commands=None,
             aliases=None,
             shortcuts=None,
-        ):
+    ):
         self.allow_redirection = allow_redirection
         if terminators is None:
             self.terminators = [';']
@@ -169,6 +169,8 @@ class StatementParser():
         """Lex a string into a list of tokens.
 
         Comments are removed, and shortcuts and aliases are expanded.
+
+        Raises ValueError if there are unclosed quotation marks.
         """
 
         # strip C-style comments
@@ -190,6 +192,8 @@ class StatementParser():
         """Tokenize the input and parse it into a Statement object, stripping
         comments, expanding aliases and shortcuts, and extracting output
         redirection directives.
+
+        Raises ValueError if there are unclosed quotation marks.
         """
 
         # handle the special case/hardcoded terminator of a blank line
@@ -310,16 +314,40 @@ class StatementParser():
         return statement
 
     def parse_command_only(self, rawinput: str) -> Statement:
-        """Partially parse input into a Statement object. The command is
-        identified, and shortcuts and aliases are expanded.
+        """Partially parse input into a Statement object.
+
+        The command is identified, and shortcuts and aliases are expanded.
         Terminators, multiline commands, and output redirection are not
         parsed.
+
+        This method is used by tab completion code and therefore must not
+        generate an exception if there are unclosed quotes.
+
+        The Statement object returned by this method can at most contained
+        values in the following attributes:
+          - raw
+          - command
+          - args
+
+        Different from parse(), this method does not remove redundant whitespace
+        within statement.args. It does however, ensure args does not have leading
+        or trailing whitespace.
         """
-        # lex the input into a list of tokens
-        tokens = self.tokenize(rawinput)
+        # expand shortcuts and aliases
+        line = self._expand(rawinput)
 
-        # parse out the command and everything else
-        (command, args) = self._command_and_args(tokens)
+        command = None
+        args = None
+        match = self.command_pattern.search(line)
+        if match:
+            # we got a match, extract the command
+            command = match.group(1)
+            # the command_pattern regex is designed to match the spaces
+            # between command and args with a second match group. Using
+            # the end of the second match group ensures that args has
+            # no leading whitespace. The rstrip() makes sure there is
+            # no trailing whitespace
+            args = line[match.end(2):].rstrip()
 
         # build the statement
         # string representation of args must be an empty string instead of
@@ -328,7 +356,6 @@ class StatementParser():
         statement.raw = rawinput
         statement.command = command
         statement.args = args
-        statement.argv = tokens
         return statement
 
     def _expand(self, line: str) -> str:
@@ -355,7 +382,7 @@ class StatementParser():
 
         # expand shortcuts
         for (shortcut, expansion) in self.shortcuts:
-            if  line.startswith(shortcut):
+            if line.startswith(shortcut):
                 # If the next character after the shortcut isn't a space, then insert one
                 shortcut_len = len(shortcut)
                 if len(line) == shortcut_len or line[shortcut_len] != ' ':
@@ -383,7 +410,7 @@ class StatementParser():
         if len(tokens) > 1:
             args = ' '.join(tokens[1:])
 
-        return (command, args)
+        return command, args
 
     @staticmethod
     def _comment_replacer(match):
@@ -400,7 +427,7 @@ class StatementParser():
         # as word breaks when they are in unquoted strings. Each run of punctuation
         # characters is treated as a single token.
 
-        :param initial_tokens: the tokens as parsed by shlex
+        :param tokens: the tokens as parsed by shlex
         :return: the punctuated tokens
         """
         punctuation = []