Merge pull request #774 from python-cmd2/completion_state

Completion state

6 files changed, 258 insertions, 224 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 59cd99fb..f6e14f11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,10 @@
 * Enhancements
     * No longer treating empty text scripts as an error condition
     * Allow dynamically extending a `cmd2.Cmd` object instance with a `do_xxx` method at runtime
+    * Choices/Completer functions can now be passed a dictionary that maps command-line tokens to their
+    argparse argument. This is helpful when one argument determines what is tab completed for another argument.
+    If these functions have an argument called `arg_tokens`, then AutoCompleter will automatically pass this
+    dictionary to them.
 
 ## 0.9.16 (August 7, 2019)
 * Bug Fixes
diff --git a/cmd2/argparse_completer.py b/cmd2/argparse_completer.py
index 41cff0dd..fb485348 100644
--- a/cmd2/argparse_completer.py
+++ b/cmd2/argparse_completer.py
@@ -7,9 +7,10 @@ See the header of argparse_custom.py for instructions on how to use these featur
 """
 
 import argparse
+import inspect
 import numbers
 import shutil
-from typing import List, Union
+from typing import Dict, List, Optional, Union
 
 from . import cmd2
 from . import utils
@@ -21,6 +22,10 @@ from .rl_utils import rl_force_redisplay
 # If no descriptive header is supplied, then this will be used instead
 DEFAULT_DESCRIPTIVE_HEADER = 'Description'
 
+# Name of the choice/completer function argument that, if present, will be passed a dictionary of
+# command line tokens up through the token being completed mapped to their argparse destination name.
+ARG_TOKENS = 'arg_tokens'
+
 
 def _single_prefix_char(token: str, parser: argparse.ArgumentParser) -> bool:
     """Returns if a token is just a single flag prefix character"""
@@ -93,28 +98,30 @@ class AutoCompleter(object):
                 self.max = self.action.nargs
 
     def __init__(self, parser: argparse.ArgumentParser, cmd2_app: cmd2.Cmd, *,
-                 token_start_index: int = 1) -> None:
+                 parent_tokens: Optional[Dict[str, List[str]]] = None) -> None:
         """
         Create an AutoCompleter
 
         :param parser: ArgumentParser instance
         :param cmd2_app: reference to the Cmd2 application that owns this AutoCompleter
-        :param token_start_index: index of the token to start parsing at
+        :param parent_tokens: optional dictionary mapping parent parsers' arg names to their tokens
+                              this is only used by AutoCompleter when recursing on subcommand parsers
+                              Defaults to None
         """
         self._parser = parser
         self._cmd2_app = cmd2_app
-        self._token_start_index = token_start_index
+
+        if parent_tokens is None:
+            parent_tokens = dict()
+        self._parent_tokens = parent_tokens
 
         self._flags = []  # all flags in this command
         self._flag_to_action = {}  # maps flags to the argparse action object
         self._positional_actions = []  # actions for positional arguments (by position index)
-
-        # maps action to subcommand autocompleter:
-        #   action -> dict(sub_command -> completer)
-        self._positional_completers = {}
+        self._subcommand_action = None  # this will be set if self._parser has subcommands
 
         # Start digging through the argparse structures.
-        #   _actions is the top level container of parameter definitions
+        #  _actions is the top level container of parameter definitions
         for action in self._parser._actions:
             # if the parameter is flag based, it will have option_strings
             if action.option_strings:
@@ -126,23 +133,13 @@ class AutoCompleter(object):
             # Otherwise this is a positional parameter
             else:
                 self._positional_actions.append(action)
-
+                # Check if this action defines subcommands
                 if isinstance(action, argparse._SubParsersAction):
-                    sub_completers = {}
-
-                    # Create an AutoCompleter for each subcommand of this command
-                    for subcmd in action.choices:
-
-                        subcmd_start = token_start_index + len(self._positional_actions)
-                        sub_completers[subcmd] = AutoCompleter(action.choices[subcmd],
-                                                               cmd2_app,
-                                                               token_start_index=subcmd_start)
-
-                    self._positional_completers[action] = sub_completers
+                    self._subcommand_action = action
 
     def complete_command(self, tokens: List[str], text: str, line: str, begidx: int, endidx: int) -> List[str]:
         """Complete the command using the argparse metadata and provided argument dictionary"""
-        if len(tokens) <= self._token_start_index:
+        if not tokens:
             return []
 
         # Count which positional argument index we're at now. Loop through all tokens on the command line so far
@@ -159,26 +156,22 @@ class AutoCompleter(object):
         # _ArgumentState of the current flag
         flag_arg_state = None
 
+        # Non-reusable flags that we've parsed
         matched_flags = []
-        consumed_arg_values = {}  # dict(arg_name -> [values, ...])
+
+        # Keeps track of arguments we've seen and any tokens they consumed
+        consumed_arg_values = dict()  # dict(arg_name -> List[tokens])
 
         def consume_argument(arg_state: AutoCompleter._ArgumentState) -> None:
             """Consuming token as an argument"""
             arg_state.count += 1
-
-            # Does this complete an option item for the flag?
-            arg_choices = self._resolve_choices_for_arg(arg_state.action)
-
-            # If the current token is in the flag argument's autocomplete list,
-            # then track that we've used it already.
-            if token in arg_choices:
-                consumed_arg_values.setdefault(arg_state.action, [])
-                consumed_arg_values[arg_state.action].append(token)
+            consumed_arg_values.setdefault(arg_state.action.dest, [])
+            consumed_arg_values[arg_state.action.dest].append(token)
 
         #############################################################################################
         # Parse all but the last token
         #############################################################################################
-        for loop_index, token in enumerate(tokens[self._token_start_index:-1]):
+        for token_index, token in enumerate(tokens[1:-1], start=1):
 
             # If we're in a positional REMAINDER arg, force all future tokens to go to that
             if pos_arg_state is not None and pos_arg_state.is_remainder:
@@ -228,13 +221,20 @@ class AutoCompleter(object):
                         action = self._flag_to_action[candidates_flags[0]]
 
                 if action is not None:
-                    # Keep track of what flags have already been used
-                    # Flags with action set to append, append_const, and count can be reused
-                    if not isinstance(action, (argparse._AppendAction,
-                                               argparse._AppendConstAction,
-                                               argparse._CountAction)):
+                    if isinstance(action, (argparse._AppendAction,
+                                           argparse._AppendConstAction,
+                                           argparse._CountAction)):
+                        # Flags with action set to append, append_const, and count can be reused
+                        # Therefore don't erase any tokens already consumed for this flag
+                        consumed_arg_values.setdefault(action.dest, [])
+                    else:
+                        # This flag is not resusable, so mark that we've seen it
                         matched_flags.extend(action.option_strings)
 
+                        # It's possible we already have consumed values for this flag if it was used
+                        # earlier in the command line. Reset them now for this use of it.
+                        consumed_arg_values[action.dest] = []
+
                     new_arg_state = AutoCompleter._ArgumentState(action)
 
                     # Keep track of this flag if it can receive arguments
@@ -242,10 +242,6 @@ class AutoCompleter(object):
                         flag_arg_state = new_arg_state
                         skip_remaining_flags = flag_arg_state.is_remainder
 
-                        # It's possible we already have consumed values for this flag if it was used
-                        # earlier in the command line. Reset them now for this use of it.
-                        consumed_arg_values[flag_arg_state.action] = []
-
             # Check if we are consuming a flag
             elif flag_arg_state is not None:
                 consume_argument(flag_arg_state)
@@ -266,11 +262,18 @@ class AutoCompleter(object):
                         action = self._positional_actions[pos_index]
 
                         # Are we at a subcommand? If so, forward to the matching completer
-                        if isinstance(action, argparse._SubParsersAction):
-                            sub_completers = self._positional_completers[action]
-                            if token in sub_completers:
-                                return sub_completers[token].complete_command(tokens, text, line,
-                                                                              begidx, endidx)
+                        if action == self._subcommand_action:
+                            if token in self._subcommand_action.choices:
+                                # Merge self._parent_tokens and consumed_arg_values
+                                parent_tokens = {**self._parent_tokens, **consumed_arg_values}
+
+                                # Include the subcommand name if its destination was set
+                                if action.dest != argparse.SUPPRESS:
+                                    parent_tokens[action.dest] = [token]
+
+                                completer = AutoCompleter(self._subcommand_action.choices[token], self._cmd2_app,
+                                                          parent_tokens=parent_tokens)
+                                return completer.complete_command(tokens[token_index:], text, line, begidx, endidx)
                             else:
                                 # Invalid subcommand entered, so no way to complete remaining tokens
                                 return []
@@ -316,9 +319,8 @@ class AutoCompleter(object):
 
         # Check if we are completing a flag's argument
         if flag_arg_state is not None:
-            consumed = consumed_arg_values.get(flag_arg_state.action, [])
             completion_results = self._complete_for_arg(flag_arg_state.action, text, line,
-                                                        begidx, endidx, consumed)
+                                                        begidx, endidx, consumed_arg_values)
 
             # If we have results, then return them
             if completion_results:
@@ -339,9 +341,8 @@ class AutoCompleter(object):
                 action = self._positional_actions[pos_index]
                 pos_arg_state = AutoCompleter._ArgumentState(action)
 
-            consumed = consumed_arg_values.get(pos_arg_state.action, [])
             completion_results = self._complete_for_arg(pos_arg_state.action, text, line,
-                                                        begidx, endidx, consumed)
+                                                        begidx, endidx, consumed_arg_values)
 
             # If we have results, then return them
             if completion_results:
@@ -411,7 +412,7 @@ class AutoCompleter(object):
 
         return completions
 
-    def complete_command_help(self, tokens: List[str], text: str, line: str, begidx: int, endidx: int) -> List[str]:
+    def complete_subcommand_help(self, tokens: List[str], text: str, line: str, begidx: int, endidx: int) -> List[str]:
         """
         Supports cmd2's help command in the completion of subcommand names
         :param tokens: command line tokens
@@ -421,121 +422,126 @@ class AutoCompleter(object):
         :param endidx: the ending index of the prefix text
         :return: List of subcommand completions
         """
-        for token in tokens[self._token_start_index:]:
-            if self._positional_completers:
-                # For now argparse only allows 1 subcommand group per level
-                # so this will only loop once.
-                for completers in self._positional_completers.values():
-                    if token in completers:
-                        return completers[token].complete_command_help(tokens, text, line, begidx, endidx)
-                    else:
-                        return utils.basic_complete(text, line, begidx, endidx, completers.keys())
+        # If our parser has subcommands, we must examine the tokens and check if they are subcommands
+        # If so, we will let the subcommand's parser handle the rest of the tokens via another AutoCompleter.
+        if self._subcommand_action is not None:
+            for token_index, token in enumerate(tokens[1:], start=1):
+                if token in self._subcommand_action.choices:
+                    completer = AutoCompleter(self._subcommand_action.choices[token], self._cmd2_app)
+                    return completer.complete_subcommand_help(tokens[token_index:], text, line, begidx, endidx)
+                elif token_index == len(tokens) - 1:
+                    # Since this is the last token, we will attempt to complete it
+                    return utils.basic_complete(text, line, begidx, endidx, self._subcommand_action.choices)
+                else:
+                    break
         return []
 
     def format_help(self, tokens: List[str]) -> str:
         """
-        Retrieve help text of a subcommand
+        Supports cmd2's help command in the retrieval of help text
         :param tokens: command line tokens
-        :return: help text of the subcommand being queried
+        :return: help text of the command being queried
         """
-        for token in tokens[self._token_start_index:]:
-            if self._positional_completers:
-                # For now argparse only allows 1 subcommand group per level
-                # so this will only loop once.
-                for completers in self._positional_completers.values():
-                    if token in completers:
-                        return completers[token].format_help(tokens)
+        # If our parser has subcommands, we must examine the tokens and check if they are subcommands
+        # If so, we will let the subcommand's parser handle the rest of the tokens via another AutoCompleter.
+        if self._subcommand_action is not None:
+            for token_index, token in enumerate(tokens[1:], start=1):
+                if token in self._subcommand_action.choices:
+                    completer = AutoCompleter(self._subcommand_action.choices[token], self._cmd2_app)
+                    return completer.format_help(tokens[token_index:])
+                else:
+                    break
         return self._parser.format_help()
 
-    def _complete_for_arg(self, arg: argparse.Action,
-                          text: str, line: str, begidx: int, endidx: int, used_values=()) -> List[str]:
-        """Tab completion routine for argparse arguments"""
-
-        # Check the arg provides choices to the user
-        if arg.choices is not None:
-            arg_choices = arg.choices
+    def _complete_for_arg(self, arg_action: argparse.Action,
+                          text: str, line: str, begidx: int, endidx: int,
+                          consumed_arg_values: Dict[str, List[str]]) -> List[str]:
+        """Tab completion routine for an argparse argument"""
+        # Check if the arg provides choices to the user
+        if arg_action.choices is not None:
+            arg_choices = arg_action.choices
         else:
-            arg_choices = getattr(arg, ATTR_CHOICES_CALLABLE, None)
+            arg_choices = getattr(arg_action, ATTR_CHOICES_CALLABLE, None)
 
         if arg_choices is None:
             return []
 
-        # Check if the argument uses a specific tab completion function to provide its choices
-        if isinstance(arg_choices, ChoicesCallable) and arg_choices.is_completer:
+        # If we are going to call a completer/choices function, then set up the common arguments
+        args = []
+        kwargs = {}
+        if isinstance(arg_choices, ChoicesCallable):
             if arg_choices.is_method:
-                results = arg_choices.to_call(self._cmd2_app, text, line, begidx, endidx)
-            else:
-                results = arg_choices.to_call(text, line, begidx, endidx)
+                args.append(self._cmd2_app)
 
-        # Otherwise use basic_complete on the choices
-        else:
-            results = utils.basic_complete(text, line, begidx, endidx,
-                                           self._resolve_choices_for_arg(arg, used_values))
+            # Check if arg_choices.to_call expects arg_tokens
+            to_call_params = inspect.signature(arg_choices.to_call).parameters
+            if ARG_TOKENS in to_call_params:
+                # Merge self._parent_tokens and consumed_arg_values
+                arg_tokens = {**self._parent_tokens, **consumed_arg_values}
 
-        return self._format_completions(arg, results)
+                # Include the token being completed
+                arg_tokens.setdefault(arg_action.dest, [])
+                arg_tokens[arg_action.dest].append(text)
 
-    def _resolve_choices_for_arg(self, arg: argparse.Action, used_values=()) -> List[str]:
-        """Retrieve a list of choices that are available for a particular argument"""
+                # Add the namespace to the keyword arguments for the function we are calling
+                kwargs[ARG_TOKENS] = arg_tokens
 
-        # Check the arg provides choices to the user
-        if arg.choices is not None:
-            arg_choices = arg.choices
+        # Check if the argument uses a specific tab completion function to provide its choices
+        if isinstance(arg_choices, ChoicesCallable) and arg_choices.is_completer:
+            args.extend([text, line, begidx, endidx])
+            results = arg_choices.to_call(*args, **kwargs)
+
+        # Otherwise use basic_complete on the choices
         else:
-            arg_choices = getattr(arg, ATTR_CHOICES_CALLABLE, None)
+            # Check if the choices come from a function
+            if isinstance(arg_choices, ChoicesCallable) and not arg_choices.is_completer:
+                arg_choices = arg_choices.to_call(*args, **kwargs)
 
-        if arg_choices is None:
-            return []
+            # Since arg_choices can be any iterable type, convert to a list
+            arg_choices = list(arg_choices)
 
-        # Check if arg_choices is a ChoicesCallable that generates a choice list
-        if isinstance(arg_choices, ChoicesCallable):
-            if arg_choices.is_completer:
-                # Tab completion routines are handled in other functions
-                return []
-            else:
-                if arg_choices.is_method:
-                    arg_choices = arg_choices.to_call(self._cmd2_app)
-                else:
-                    arg_choices = arg_choices.to_call()
+            # If these choices are numbers, and have not yet been sorted, then sort them now
+            if not self._cmd2_app.matches_sorted and all(isinstance(x, numbers.Number) for x in arg_choices):
+                arg_choices.sort()
+                self._cmd2_app.matches_sorted = True
 
-        # Since arg_choices can be any iterable type, convert to a list
-        arg_choices = list(arg_choices)
+            # Since choices can be various types like int, we must convert them to strings
+            for index, choice in enumerate(arg_choices):
+                if not isinstance(choice, str):
+                    arg_choices[index] = str(choice)
 
-        # If these choices are numbers, and have not yet been sorted, then sort them now
-        if not self._cmd2_app.matches_sorted and all(isinstance(x, numbers.Number) for x in arg_choices):
-            arg_choices.sort()
-            self._cmd2_app.matches_sorted = True
+            # Filter out arguments we already used
+            used_values = consumed_arg_values.get(arg_action.dest, [])
+            arg_choices = [choice for choice in arg_choices if choice not in used_values]
 
-        # Since choices can be various types like int, we must convert them to strings
-        for index, choice in enumerate(arg_choices):
-            if not isinstance(choice, str):
-                arg_choices[index] = str(choice)
+            # Do tab completion on the choices
+            results = utils.basic_complete(text, line, begidx, endidx, arg_choices)
 
-        # Filter out arguments we already used
-        return [choice for choice in arg_choices if choice not in used_values]
+        return self._format_completions(arg_action, results)
 
     @staticmethod
-    def _print_arg_hint(arg: argparse.Action) -> None:
+    def _print_arg_hint(arg_action: argparse.Action) -> None:
         """Print argument hint to the terminal when tab completion results in no results"""
 
         # Check if hinting is disabled
-        suppress_hint = getattr(arg, ATTR_SUPPRESS_TAB_HINT, False)
-        if suppress_hint or arg.help == argparse.SUPPRESS or arg.dest == argparse.SUPPRESS:
+        suppress_hint = getattr(arg_action, ATTR_SUPPRESS_TAB_HINT, False)
+        if suppress_hint or arg_action.help == argparse.SUPPRESS or arg_action.dest == argparse.SUPPRESS:
             return
 
         # Check if this is a flag
-        if arg.option_strings:
-            flags = ', '.join(arg.option_strings)
-            param = ' ' + str(arg.dest).upper()
+        if arg_action.option_strings:
+            flags = ', '.join(arg_action.option_strings)
+            param = ' ' + str(arg_action.dest).upper()
             prefix = '{}{}'.format(flags, param)
 
         # Otherwise this is a positional
         else:
-            prefix = '{}'.format(str(arg.dest).upper())
+            prefix = '{}'.format(str(arg_action.dest).upper())
 
         prefix = '  {0: <{width}}    '.format(prefix, width=20)
         pref_len = len(prefix)
 
-        help_text = '' if arg.help is None else arg.help
+        help_text = '' if arg_action.help is None else arg_action.help
         help_lines = help_text.splitlines()
 
         if len(help_lines) == 1:
diff --git a/cmd2/argparse_custom.py b/cmd2/argparse_custom.py
index 2a7be287..940d6064 100644
--- a/cmd2/argparse_custom.py
+++ b/cmd2/argparse_custom.py
@@ -44,7 +44,7 @@ Tab Completion:
         generated when the user hits tab.
 
         Example:
-            def my_choices_function):
+            def my_choices_function():
                 ...
                 return my_generated_list
 
@@ -102,6 +102,20 @@ Tab Completion:
         set_completer_function(action, func)
         set_completer_method(action, method)
 
+    There are times when what's being tab completed is determined by a previous argument on the command line.
+    In theses cases, Autocompleter can pass a dictionary that maps the command line tokens up through the one
+    being completed to their argparse argument name. To receive this dictionary, your choices/completer function
+    should have an argument called arg_tokens.
+
+        Example:
+            def my_choices_method(self, arg_tokens)
+            def my_completer_method(self, text, line, begidx, endidx, arg_tokens)
+
+    All values of the arg_tokens dictionary are lists, even if a particular argument expects only 1 token. Since
+    AutoCompleter is for tab completion, it does not convert the tokens to their actual argument types or validate
+    their values. All tokens are stored in the dictionary as the raw strings provided on the command line. It is up to
+    the developer to determine if the user entered the correct argument type (e.g. int) and validate their values.
+
 CompletionItem Class:
     This class was added to help in cases where uninformative data is being tab completed. For instance,
     tab completing ID numbers isn't very helpful to a user without context. Returning a list of CompletionItems
@@ -138,7 +152,7 @@ CompletionItem Class:
     To use CompletionItems, just return them from your choices or completer functions.
 
     To avoid printing a ton of information to the screen at once when a user presses tab, there is
-    a maximum threshold for the number of CompletionItems that will be shown. It's value is defined
+    a maximum threshold for the number of CompletionItems that will be shown. Its value is defined
     in cmd2.Cmd.max_completion_items. It defaults to 50, but can be changed. If the number of completion
     suggestions exceeds this number, they will be displayed in the typical columnized format and will
     not include the description value of the CompletionItems.
@@ -159,10 +173,9 @@ argparse.ArgumentParser._match_argument - adds support to for nargs ranges
 import argparse
 import re
 import sys
-
 # noinspection PyUnresolvedReferences,PyProtectedMember
 from argparse import ZERO_OR_MORE, ONE_OR_MORE, ArgumentError, _
-from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
+from typing import Callable, Optional, Tuple, Union
 
 from .ansi import ansi_aware_write, style_error
 
@@ -272,24 +285,22 @@ def _set_choices_callable(action: argparse.Action, choices_callable: ChoicesCall
     setattr(action, ATTR_CHOICES_CALLABLE, choices_callable)
 
 
-def set_choices_function(action: argparse.Action, choices_function: Callable[[], Iterable[Any]]) -> None:
+def set_choices_function(action: argparse.Action, choices_function: Callable) -> None:
     """Set choices_function on an argparse action"""
     _set_choices_callable(action, ChoicesCallable(is_method=False, is_completer=False, to_call=choices_function))
 
 
-def set_choices_method(action: argparse.Action, choices_method: Callable[[Any], Iterable[Any]]) -> None:
+def set_choices_method(action: argparse.Action, choices_method: Callable) -> None:
     """Set choices_method on an argparse action"""
     _set_choices_callable(action, ChoicesCallable(is_method=True, is_completer=False, to_call=choices_method))
 
 
-def set_completer_function(action: argparse.Action,
-                           completer_function: Callable[[str, str, int, int], List[str]]) -> None:
+def set_completer_function(action: argparse.Action, completer_function: Callable) -> None:
     """Set completer_function on an argparse action"""
     _set_choices_callable(action, ChoicesCallable(is_method=False, is_completer=True, to_call=completer_function))
 
 
-def set_completer_method(action: argparse.Action,
-                         completer_method: Callable[[Any, str, str, int, int], List[str]]) -> None:
+def set_completer_method(action: argparse.Action, completer_method: Callable) -> None:
     """Set completer_method on an argparse action"""
     _set_choices_callable(action, ChoicesCallable(is_method=True, is_completer=True, to_call=completer_method))
 
@@ -305,10 +316,10 @@ orig_actions_container_add_argument = argparse._ActionsContainer.add_argument
 
 def _add_argument_wrapper(self, *args,
                           nargs: Union[int, str, Tuple[int], Tuple[int, int], None] = None,
-                          choices_function: Optional[Callable[[], Iterable[Any]]] = None,
-                          choices_method: Optional[Callable[[Any], Iterable[Any]]] = None,
-                          completer_function: Optional[Callable[[str, str, int, int], List[str]]] = None,
-                          completer_method: Optional[Callable[[Any, str, str, int, int], List[str]]] = None,
+                          choices_function: Optional[Callable] = None,
+                          choices_method: Optional[Callable] = None,
+                          completer_function: Optional[Callable] = None,
+                          completer_method: Optional[Callable] = None,
                           suppress_tab_hint: bool = False,
                           descriptive_header: Optional[str] = None,
                           **kwargs) -> argparse.Action:
diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 610ec897..69de58b0 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -119,6 +119,9 @@ CMD_ATTR_HELP_CATEGORY = 'help_category'
 # The argparse parser for the command
 CMD_ATTR_ARGPARSER = 'argparser'
 
+# Whether or not tokens are unquoted before sending to argparse
+CMD_ATTR_PRESERVE_QUOTES = 'preserve_quotes'
+
 
 def categorize(func: Union[Callable, Iterable[Callable]], category: str) -> None:
     """Categorize a function.
@@ -225,8 +228,9 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, *,
         # Set the command's help text as argparser.description (which can be None)
         cmd_wrapper.__doc__ = argparser.description
 
-        # Mark this function as having an argparse ArgumentParser
+        # Set some custom attributes for this command
         setattr(cmd_wrapper, CMD_ATTR_ARGPARSER, argparser)
+        setattr(cmd_wrapper, CMD_ATTR_PRESERVE_QUOTES, preserve_quotes)
 
         return cmd_wrapper
 
@@ -283,8 +287,9 @@ def with_argparser(argparser: argparse.ArgumentParser, *,
         # Set the command's help text as argparser.description (which can be None)
         cmd_wrapper.__doc__ = argparser.description
 
-        # Mark this function as having an argparse ArgumentParser
+        # Set some custom attributes for this command
         setattr(cmd_wrapper, CMD_ATTR_ARGPARSER, argparser)
+        setattr(cmd_wrapper, CMD_ATTR_PRESERVE_QUOTES, preserve_quotes)
 
         return cmd_wrapper
 
@@ -1431,7 +1436,8 @@ class Cmd(cmd.Cmd):
                 if func is not None and argparser is not None:
                     import functools
                     compfunc = functools.partial(self._autocomplete_default,
-                                                 argparser=argparser)
+                                                 argparser=argparser,
+                                                 preserve_quotes=getattr(func, CMD_ATTR_PRESERVE_QUOTES))
                 else:
                     compfunc = self.completedefault
 
@@ -1588,13 +1594,17 @@ class Cmd(cmd.Cmd):
             self.pexcept(e)
             return None
 
-    def _autocomplete_default(self, text: str, line: str, begidx: int, endidx: int,
-                              argparser: argparse.ArgumentParser) -> List[str]:
+    def _autocomplete_default(self, text: str, line: str, begidx: int, endidx: int, *,
+                              argparser: argparse.ArgumentParser, preserve_quotes: bool) -> List[str]:
         """Default completion function for argparse commands"""
         from .argparse_completer import AutoCompleter
         completer = AutoCompleter(argparser, self)
-        tokens, _ = self.tokens_for_completion(line, begidx, endidx)
-        return completer.complete_command(tokens, text, line, begidx, endidx)
+        tokens, raw_tokens = self.tokens_for_completion(line, begidx, endidx)
+
+        # To have tab-completion parsing match command line parsing behavior,
+        # use preserve_quotes to determine if we parse the quoted or unquoted tokens.
+        tokens_to_parse = raw_tokens if preserve_quotes else tokens
+        return completer.complete_command(tokens_to_parse, text, line, begidx, endidx)
 
     def get_names(self):
         """Return an alphabetized list of names comprising the attributes of the cmd2 class instance."""
@@ -2662,42 +2672,27 @@ class Cmd(cmd.Cmd):
         strs_to_match = list(topics | visible_commands)
         return utils.basic_complete(text, line, begidx, endidx, strs_to_match)
 
-    def complete_help_subcommand(self, text: str, line: str, begidx: int, endidx: int) -> List[str]:
+    def complete_help_subcommand(self, text: str, line: str, begidx: int, endidx: int,
+                                 arg_tokens: Dict[str, List[str]]) -> List[str]:
         """Completes the subcommand argument of help"""
 
-        # Get all tokens through the one being completed
-        tokens, _ = self.tokens_for_completion(line, begidx, endidx)
-
-        if not tokens:
-            return []
-
-        # Must have at least 3 args for 'help command subcommand'
-        if len(tokens) < 3:
+        # Make sure we have a command whose subcommands we will complete
+        command = arg_tokens['command'][0]
+        if not command:
             return []
 
-        # Find where the command is by skipping past any flags
-        cmd_index = 1
-        for cur_token in tokens[cmd_index:]:
-            if not cur_token.startswith('-'):
-                break
-            cmd_index += 1
-
-        if cmd_index >= len(tokens):
-            return []
-
-        command = tokens[cmd_index]
-        matches = []
-
         # Check if this command uses argparse
         func = self.cmd_func(command)
         argparser = getattr(func, CMD_ATTR_ARGPARSER, None)
+        if func is None or argparser is None:
+            return []
 
-        if func is not None and argparser is not None:
-            from .argparse_completer import AutoCompleter
-            completer = AutoCompleter(argparser, self)
-            matches = completer.complete_command_help(tokens[cmd_index:], text, line, begidx, endidx)
+        # Combine the command and its subcommand tokens for the AutoCompleter
+        tokens = [command] + arg_tokens['subcommand']
 
-        return matches
+        from .argparse_completer import AutoCompleter
+        completer = AutoCompleter(argparser, self)
+        return completer.complete_subcommand_help(tokens, text, line, begidx, endidx)
 
     help_parser = Cmd2ArgumentParser(description="List available commands or provide "
                                                  "detailed help for a specific command")
@@ -2963,14 +2958,11 @@ class Cmd(cmd.Cmd):
                 choice = int(response)
                 if choice < 1:
                     raise IndexError
-                result = fulloptions[choice - 1][0]
-                break
+                return fulloptions[choice - 1][0]
             except (ValueError, IndexError):
                 self.poutput("{!r} isn't a valid choice. Pick a number between 1 and {}:".format(
                     response, len(fulloptions)))
 
-        return result
-
     def _get_read_only_settings(self) -> str:
         """Return a summary report of read-only settings which the user cannot modify at runtime.
 
@@ -4125,8 +4117,7 @@ class Cmd(cmd.Cmd):
             if getattr(func, CMD_ATTR_HELP_CATEGORY, None) == category:
                 self.disable_command(cmd_name, message_to_print)
 
-    # noinspection PyUnusedLocal
-    def _report_disabled_command_usage(self, *args, message_to_print: str, **kwargs) -> None:
+    def _report_disabled_command_usage(self, *_args, message_to_print: str, **_kwargs) -> None:
         """
         Report when a disabled command has been run or had help called on it
         :param args: not used
diff --git a/tests/test_argparse_completer.py b/tests/test_argparse_completer.py
index 68a2320c..788a7e59 100644
--- a/tests/test_argparse_completer.py
+++ b/tests/test_argparse_completer.py
@@ -41,6 +41,18 @@ def completer_function(text: str, line: str, begidx: int, endidx: int) -> List[s
     return basic_complete(text, line, begidx, endidx, completions_from_function)
 
 
+def choices_takes_arg_tokens(arg_tokens: argparse.Namespace) -> List[str]:
+    """Choices function that receives arg_tokens from AutoCompleter"""
+    return [arg_tokens['parent_arg'][0], arg_tokens['subcommand'][0]]
+
+
+def completer_takes_arg_tokens(text: str, line: str, begidx: int, endidx: int,
+                               arg_tokens: argparse.Namespace) -> List[str]:
+    """Completer function that receives arg_tokens from AutoCompleter"""
+    match_against = [arg_tokens['parent_arg'][0], arg_tokens['subcommand'][0]]
+    return basic_complete(text, line, begidx, endidx, match_against)
+
+
 # noinspection PyMethodMayBeStatic,PyUnusedLocal
 class AutoCompleteTester(cmd2.Cmd):
     """Cmd2 app that exercises AutoCompleter class"""
@@ -50,50 +62,21 @@ class AutoCompleteTester(cmd2.Cmd):
     ############################################################################################################
     # Begin code related to help and command name completion
     ############################################################################################################
-    def _music_create(self, args: argparse.Namespace) -> None:
-        """Implements the 'music create' command"""
-        self.poutput('music create')
-
-    def _music_create_jazz(self, args: argparse.Namespace) -> None:
-        """Implements the 'music create jazz' command"""
-        self.poutput('music create jazz')
-
-    def _music_create_rock(self, args: argparse.Namespace) -> None:
-        """Implements the 'music create rock' command"""
-        self.poutput('music create rock')
-
     # Top level parser for music command
     music_parser = Cmd2ArgumentParser(description='Manage music', prog='music')
 
     # Add subcommands to music
     music_subparsers = music_parser.add_subparsers()
-
-    # music -> create
     music_create_parser = music_subparsers.add_parser('create', help='Create music')
-    music_create_parser.set_defaults(func=_music_create)
 
     # Add subcommands to music -> create
     music_create_subparsers = music_create_parser.add_subparsers()
-
-    # music -> create -> jazz
     music_create_jazz_parser = music_create_subparsers.add_parser('jazz', help='Create jazz')
-    music_create_jazz_parser.set_defaults(func=_music_create_jazz)
-
-    # music -> create -> rock
     music_create_rock_parser = music_create_subparsers.add_parser('rock', help='Create rocks')
-    music_create_rock_parser.set_defaults(func=_music_create_rock)
 
     @with_argparser(music_parser)
     def do_music(self, args: argparse.Namespace) -> None:
-        """Music command"""
-        func = getattr(args, 'func', None)
-        if func is not None:
-            # Call whatever subcommand function was selected
-            func(self, args)
-        else:
-            # No subcommand was provided, so call help
-            # noinspection PyTypeChecker
-            self.do_help('music')
+        pass
 
     ############################################################################################################
     # Begin code related to flag completion
@@ -227,6 +210,26 @@ class AutoCompleteTester(cmd2.Cmd):
     def do_hint(self, args: argparse.Namespace) -> None:
         pass
 
+    ############################################################################################################
+    # Begin code related to receiving arg_tokens
+    ############################################################################################################
+    arg_tokens_parser = Cmd2ArgumentParser()
+    arg_tokens_parser.add_argument('parent_arg', help='arg from a parent parser')
+
+    # Create a subcommand for to exercise receiving parent_tokens and subcommand name in arg_tokens
+    arg_tokens_subparser = arg_tokens_parser.add_subparsers(dest='subcommand')
+    arg_tokens_subcmd_parser = arg_tokens_subparser.add_parser('subcmd')
+
+    arg_tokens_subcmd_parser.add_argument('choices_pos', choices_function=choices_takes_arg_tokens)
+    arg_tokens_subcmd_parser.add_argument('completer_pos', completer_function=completer_takes_arg_tokens)
+
+    # Used to override parent_arg in arg_tokens_parser
+    arg_tokens_subcmd_parser.add_argument('--parent_arg')
+
+    @with_argparser(arg_tokens_parser)
+    def do_arg_tokens(self, args: argparse.Namespace) -> None:
+        pass
+
 
 @pytest.fixture
 def ac_app():
@@ -253,7 +256,9 @@ def test_help(ac_app, command):
     ('music', 'creab', []),
     ('music create', '', ['jazz', 'rock']),
     ('music crea', 'jazz', []),
-    ('music create', 'foo', [])
+    ('music create', 'foo', []),
+    ('fake create', '', []),
+    ('music fake', '', [])
 ])
 def test_complete_help(ac_app, command, text, completions):
     line = 'help {} {}'.format(command, text)
@@ -718,6 +723,31 @@ Hint:
 '''
 
 
+@pytest.mark.parametrize('command_and_args, completions', [
+    # Exercise a choices function that receives arg_tokens dictionary
+    ('arg_tokens choice subcmd', ['choice', 'subcmd']),
+
+    # Exercise a completer that receives arg_tokens dictionary
+    ('arg_tokens completer subcmd fake', ['completer', 'subcmd']),
+
+    # Exercise overriding parent_arg from the subcommand
+    ('arg_tokens completer subcmd --parent_arg override fake', ['override', 'subcmd'])
+])
+def test_arg_tokens(ac_app, command_and_args, completions):
+    text = ''
+    line = '{} {}'.format(command_and_args, text)
+    endidx = len(line)
+    begidx = endidx - len(text)
+
+    first_match = complete_tester(text, line, begidx, endidx, ac_app)
+    if completions:
+        assert first_match is not None
+    else:
+        assert first_match is None
+
+    assert ac_app.completion_matches == sorted(completions, key=ac_app.default_sort_key)
+
+
 def test_single_prefix_char():
     from cmd2.argparse_completer import _single_prefix_char
     parser = Cmd2ArgumentParser(prefix_chars='-+')
@@ -767,5 +797,5 @@ def test_complete_command_help_no_tokens(ac_app):
     parser = Cmd2ArgumentParser()
     ac = AutoCompleter(parser, ac_app)
 
-    completions = ac.complete_command_help(tokens=[], text='', line='', begidx=0, endidx=0)
+    completions = ac.complete_subcommand_help(tokens=[], text='', line='', begidx=0, endidx=0)
     assert not completions
diff --git a/tests/test_completion.py b/tests/test_completion.py
index cf5dcf75..fb0d74e0 100755
--- a/tests/test_completion.py
+++ b/tests/test_completion.py
@@ -1179,22 +1179,14 @@ def test_cmd2_help_subcommand_completion_with_flags_before_command(scu_app):
     first_match = complete_tester(text, line, begidx, endidx, scu_app)
     assert first_match is not None and scu_app.completion_matches == ['bar', 'foo', 'sport']
 
-def test_complete_help_subcommand_with_no_command(scu_app):
-    # No command because not enough tokens
+def test_complete_help_subcommand_with_blank_command(scu_app):
     text = ''
-    line = 'help '
+    line = 'help "" {}'.format(text)
     endidx = len(line)
     begidx = endidx - len(text)
 
-    assert not scu_app.complete_help_subcommand(text, line, begidx, endidx)
-
-    # No command because everything is a flag
-    text = '-v'
-    line = 'help -f -v'
-    endidx = len(line)
-    begidx = endidx - len(text)
-
-    assert not scu_app.complete_help_subcommand(text, line, begidx, endidx)
+    first_match = complete_tester(text, line, begidx, endidx, scu_app)
+    assert first_match is None and not scu_app.completion_matches
 
 
 def test_cmd2_help_subcommand_completion_nomatch_scu(scu_app):