First stage of refactoring cmd2.parseline() for tab completion

4 files changed, 122 insertions, 63 deletions

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index cd80970b..89e07802 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -1588,6 +1588,11 @@ class Cmd(cmd.Cmd):
 
                 # Parse the command line
                 command, args, expanded_line = self.parseline(line)
+                
+                # use these lines instead of the one above
+                # statement = self.command_parser.parse_command_only(line)
+                # command = statement.command
+                # expanded_line = statement.command_and_args
 
                 # We overwrote line with a properly formatted but fully stripped version
                 # Restore the end spaces since line is only supposed to be lstripped when
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 9030a5f8..45715b32 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -10,6 +10,14 @@ import cmd2
 
 BLANK_LINE = '\n'
 
+def _comment_replacer(match):
+    s = match.group(0)
+    if s.startswith('/'):
+        # treat the removed comment as an empty string
+        return ''
+    else:
+        return s
+
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
     
@@ -32,6 +40,11 @@ class Statement(str):
         self.pipeTo = None
         self.output = None
         self.outputTo = None
+    
+    @property
+    def command_and_args(self):
+        """Combine command and args with a space separating them"""
+        return '{} {}'.format('' if self.command is None else self.command, self.args).strip()
 
 class StatementParser():
     """Parse raw text into command components.
@@ -56,40 +69,28 @@ class StatementParser():
         self.aliases = aliases
         self.shortcuts = shortcuts
 
-    def parse(self, rawinput: str) -> Statement:
-        # strip C-style comments
-        # shlex will handle the python/shell style comments for us
-        def replacer(match):
-                s = match.group(0)
-                if s.startswith('/'):
-                    # treat the removed comment as an empty string
-                    return ''
-                else:
-                    return s
-        pattern = re.compile(
-            #r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+        self.comment_pattern = re.compile(
             r'/\*.*?(\*/|$)|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE
         )
-        rawinput = re.sub(pattern, replacer, rawinput)
-        line = rawinput
 
+    def parse(self, rawinput: str) -> Statement:
+        """Parse input into a Statement object, stripping comments, expanding
+        aliases and shortcuts, and extracting output redirection directives.
+        """
+        # strip C-style comments
+        # shlex will handle the python/shell style comments for us
+        # save rawinput for later
+        rawinput = re.sub(self.comment_pattern, _comment_replacer, rawinput)
+        # we are going to modify line, so create a copy of the raw input
+        line = rawinput
         command = None
         args = ''
 
         # expand shortcuts, have to do this first because
         # a shortcut can expand into multiple tokens, ie '!ls' becomes
         # 'shell ls'
-        for (shortcut, expansion) in self.shortcuts:
-            if  line.startswith(shortcut):
-                # If the next character after the shortcut isn't a space, then insert one
-                shortcut_len = len(shortcut)
-                if len(line) == shortcut_len or line[shortcut_len] != ' ':
-                    expansion += ' '
-
-                # Expand the shortcut
-                line = line.replace(shortcut, expansion, 1)
-                break
+        line = self.expand_shortcuts(line)
 
         # handle the special case/hardcoded terminator of a blank line
         # we have to do this before we shlex on whitespace because it
@@ -98,10 +99,12 @@ class StatementParser():
         if line[-1:] == BLANK_LINE:
             terminator = BLANK_LINE
 
+        # split the input on whitespace
         s = shlex.shlex(line, posix=False)
         s.whitespace_split = True
         tokens = self.split_on_punctuation(list(s))
 
+        # expand aliases
         if tokens:
             command_to_expand = tokens[0]
             tokens[0] = self.expand_aliases(command_to_expand)
@@ -211,6 +214,59 @@ class StatementParser():
         result.multilineCommand = multilineCommand
         return result
 
+    def parse_command_only(self, rawinput: str) -> Statement:
+        """Partially parse input into a Statement object. The command is
+        identified, and shortcuts and aliases are expanded.
+        Terminators, multiline commands, and output redirection are not
+        parsed.
+        """
+        # strip C-style comments
+        # shlex will handle the python/shell style comments for us
+        # save rawinput for later
+        rawinput = re.sub(self.comment_pattern, _comment_replacer, rawinput)
+        # we are going to modify line, so create a copy of the raw input
+        line = rawinput
+        command = None
+        args = ''
+
+        # expand shortcuts, have to do this first because
+        # a shortcut can expand into multiple tokens, ie '!ls' becomes
+        # 'shell ls'
+        line = self.expand_shortcuts(line)
+
+        # split the input on whitespace
+        s = shlex.shlex(line, posix=False)
+        s.whitespace_split = True
+        tokens = self.split_on_punctuation(list(s))
+
+        # expand aliases
+        if tokens:
+            command_to_expand = tokens[0]
+            tokens[0] = self.expand_aliases(command_to_expand)
+
+        (command, args) = self._command_and_args(tokens)
+
+        # build Statement object
+        result = Statement(args)
+        result.raw = rawinput
+        result.command = command
+        result.args = args
+        return result
+
+    def expand_shortcuts(self, line: str) -> str:
+        """Expand shortcuts at the beginning of input."""
+        for (shortcut, expansion) in self.shortcuts:
+            if  line.startswith(shortcut):
+                # If the next character after the shortcut isn't a space, then insert one
+                shortcut_len = len(shortcut)
+                if len(line) == shortcut_len or line[shortcut_len] != ' ':
+                    expansion += ' '
+
+                # Expand the shortcut
+                line = line.replace(shortcut, expansion, 1)
+                break
+        return line
+
     def expand_aliases(self, command: str) -> str:
         """Given a command, expand any aliases for the command"""
         # make a copy of aliases so we can edit it
diff --git a/tests/test_completion.py b/tests/test_completion.py
index cf45f281..f916f2e8 100644
--- a/tests/test_completion.py
+++ b/tests/test_completion.py
@@ -581,44 +581,6 @@ def test_tokens_for_completion_redirect_off(cmd2_app):
     assert expected_tokens == tokens
     assert expected_raw_tokens == raw_tokens
 
-def test_parseline_command_and_args(cmd2_app):
-    line = 'help history'
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'help'
-    assert args == 'history'
-    assert line == out_line
-
-def test_parseline_emptyline(cmd2_app):
-    line = ''
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command is None
-    assert args is None
-    assert line is out_line
-
-def test_parseline_strips_line(cmd2_app):
-    line = '  help history  '
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'help'
-    assert args == 'history'
-    assert line.strip() == out_line
-
-def test_parseline_expands_alias(cmd2_app):
-    # Create the alias
-    cmd2_app.do_alias(['fake', 'pyscript'])
-
-    line = 'fake foobar.py'
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'pyscript'
-    assert args == 'foobar.py'
-    assert line.replace('fake', 'pyscript') == out_line
-
-def test_parseline_expands_shortcuts(cmd2_app):
-    line = '!cat foobar.txt'
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'shell'
-    assert args == 'cat foobar.txt'
-    assert line.replace('!', 'shell ') == out_line
-
 def test_add_opening_quote_basic_no_text(cmd2_app):
     text = ''
     line = 'test_basic {}'.format(text)
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 1913938e..d7a872b3 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -19,7 +19,7 @@ def parser():
         redirection_chars=['|', '<', '>'],        
         terminators = [';'],
         multilineCommands = ['multiline'],
-        aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline'},
+        aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},
         shortcuts = [('?', 'help'), ('!', 'shell')]
     )
     return parser
@@ -27,6 +27,8 @@ def parser():
 def test_parse_empty_string(parser):
     statement = parser.parse('')
     assert not statement.command
+    assert not statement.args
+    assert statement.raw == ''
 
 @pytest.mark.parametrize('tokens,command,args', [
     ( [], None, ''),
@@ -287,3 +289,37 @@ def test_alias_on_multiline_command(parser):
     assert statement.command == 'multiline'
     assert statement.args == 'has > inside an unfinished command'
     assert not statement.terminator
+
+def test_parse_command_only_command_and_args(parser):
+    line = 'help history'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'help'
+    assert statement.args == 'history'
+    assert statement.command_and_args == line
+
+def test_parse_command_only_emptyline(parser):
+    line = ''
+    statement = parser.parse_command_only(line)
+    assert statement.command is None
+    assert statement.args is ''
+    assert statement.command_and_args is line
+
+def test_parse_command_only_strips_line(parser):
+    line = '  help history  '
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'help'
+    assert statement.args == 'history'
+    assert statement.command_and_args == line.strip()
+
+def test_parse_command_only_expands_alias(parser):
+    line = 'fake foobar.py'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'pyscript'
+    assert statement.args == 'foobar.py'
+
+def test_parse_command_only_expands_shortcuts(parser):
+    line = '!cat foobar.txt'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'shell'
+    assert statement.args == 'cat foobar.txt'
+    assert statement.command_and_args == line.replace('!', 'shell ')