Merge branch 'master' into delete_optparse

# Conflicts:
#	cmd2.py

5 files changed, 534 insertions, 17 deletions

diff --git a/cmd2.py b/cmd2.py
index f2dc13f5..5d15f884 100755
--- a/cmd2.py
+++ b/cmd2.py
@@ -55,7 +55,7 @@ import pyperclip
 # Collection is a container that is sizable and iterable
 # It was introduced in Python 3.6. We will try to import it, otherwise use our implementation
 try:
-    from collections.abc import Collection
+    from collections.abc import Collection, Iterable
 except ImportError:
 
     if six.PY3:
@@ -78,7 +78,6 @@ except ImportError:
                     return True
             return NotImplemented
 
-
 # Newer versions of pyperclip are released as a single file, but older versions had a more complicated structure
 try:
     from pyperclip.exceptions import PyperclipException
@@ -112,6 +111,11 @@ if sys.version_info < (3, 5):
 else:
     from contextlib import redirect_stdout, redirect_stderr
 
+if sys.version_info > (3, 0):
+    from io import StringIO  # Python3
+else:
+    from io import BytesIO as StringIO  # Python2
+
 # Detect whether IPython is installed to determine if the built-in "ipy" command should be included
 ipython_available = True
 try:
@@ -205,6 +209,25 @@ STRIP_QUOTES_FOR_NON_POSIX = True
 QUOTES = ['"', "'"]
 REDIRECTION_CHARS = ['|', '<', '>']
 
+# optional attribute, when tagged on a function, allows cmd2 to categorize commands
+HELP_CATEGORY = 'help_category'
+HELP_SUMMARY = 'help_summary'
+
+
+def categorize(func, category):
+    """Categorize a function.
+
+    The help command output will group this function under the specified category heading
+
+    :param func: Union[Callable, Iterable] - function to categorize
+    :param category: str - category to put it in
+    """
+    if isinstance(func, Iterable):
+        for item in func:
+            setattr(item, HELP_CATEGORY, category)
+    else:
+        setattr(func, HELP_CATEGORY, category)
+
 
 def set_posix_shlex(val):
     """ Allows user of cmd2 to choose between POSIX and non-POSIX splitting of args for decorated commands.
@@ -266,6 +289,14 @@ def parse_quoted_string(cmdline):
     return lexed_arglist
 
 
+def with_category(category):
+    """A decorator to apply a category to a command function"""
+    def cat_decorator(func):
+        categorize(func, category)
+        return func
+    return cat_decorator
+
+
 def with_argument_list(func):
     """A decorator to alter the arguments passed to a do_* cmd2
     method. Default passes a string of whatever the user typed.
@@ -304,6 +335,9 @@ def with_argparser_and_unknown_args(argparser):
         if argparser.description is None and func.__doc__:
             argparser.description = func.__doc__
 
+        if func.__doc__:
+            setattr(cmd_wrapper, HELP_SUMMARY, func.__doc__)
+
         cmd_wrapper.__doc__ = argparser.format_help()
 
         # Mark this function as having an argparse ArgumentParser (used by do_help)
@@ -343,6 +377,9 @@ def with_argparser(argparser):
         if argparser.description is None and func.__doc__:
             argparser.description = func.__doc__
 
+        if func.__doc__:
+            setattr(cmd_wrapper, HELP_SUMMARY, func.__doc__)
+
         cmd_wrapper.__doc__ = argparser.format_help()
 
         # Mark this function as having an argparse ArgumentParser (used by do_help)
@@ -2700,7 +2737,10 @@ Usage:  Usage: unalias [-a] name [name ...]
     @with_argument_list
     def do_help(self, arglist):
         """List available commands with "help" or detailed help with "help cmd"."""
-        if arglist:
+        if not arglist or (len(arglist) == 1 and arglist[0] in ('--verbose', '-v')):
+            verbose = len(arglist) == 1 and arglist[0] in ('--verbose', '-v')
+            self._help_menu(verbose)
+        else:
             # Getting help for a specific command
             funcname = self._func_named(arglist[0])
             if funcname:
@@ -2721,11 +2761,8 @@ Usage:  Usage: unalias [-a] name [name ...]
             else:
                 # This could be a help topic
                 cmd.Cmd.do_help(self, arglist[0])
-        else:
-            # Show a menu of what commands help can be gotten for
-            self._help_menu()
 
-    def _help_menu(self):
+    def _help_menu(self, verbose=False):
         """Show a list of commands which help can be displayed for.
         """
         # Get a sorted list of help topics
@@ -2738,21 +2775,107 @@ Usage:  Usage: unalias [-a] name [name ...]
 
         cmds_doc = []
         cmds_undoc = []
+        cmds_cats = {}
 
         for command in visible_commands:
-            if command in help_topics:
-                cmds_doc.append(command)
-                help_topics.remove(command)
-            elif getattr(self, self._func_named(command)).__doc__:
-                cmds_doc.append(command)
+            if command in help_topics or getattr(self, self._func_named(command)).__doc__:
+                if command in help_topics:
+                    help_topics.remove(command)
+                if hasattr(getattr(self, self._func_named(command)), HELP_CATEGORY):
+                    category = getattr(getattr(self, self._func_named(command)), HELP_CATEGORY)
+                    cmds_cats.setdefault(category, [])
+                    cmds_cats[category].append(command)
+                else:
+                    cmds_doc.append(command)
             else:
                 cmds_undoc.append(command)
 
-        self.poutput("%s\n" % str(self.doc_leader))
-        self.print_topics(self.doc_header, cmds_doc, 15, 80)
+        if len(cmds_cats) == 0:
+            # No categories found, fall back to standard behavior
+            self.poutput("{}\n".format(str(self.doc_leader)))
+            self._print_topics(self.doc_header, cmds_doc, verbose)
+        else:
+            # Categories found, Organize all commands by category
+            self.poutput('{}\n'.format(str(self.doc_leader)))
+            self.poutput('{}\n\n'.format(str(self.doc_header)))
+            for category in sorted(cmds_cats.keys()):
+                self._print_topics(category, cmds_cats[category], verbose)
+            self._print_topics('Other', cmds_doc, verbose)
+
         self.print_topics(self.misc_header, help_topics, 15, 80)
         self.print_topics(self.undoc_header, cmds_undoc, 15, 80)
 
+    def _print_topics(self, header, cmds, verbose):
+        """Customized version of print_topics that can switch between verbose or traditional output"""
+        if cmds:
+            if not verbose:
+                self.print_topics(header, cmds, 15, 80)
+            else:
+                self.stdout.write('{}\n'.format(str(header)))
+                widest = 0
+                # measure the commands
+                for command in cmds:
+                    width = len(command)
+                    if width > widest:
+                        widest = width
+                # add a 4-space pad
+                widest += 4
+                if widest < 20:
+                    widest = 20
+
+                if self.ruler:
+                    self.stdout.write('{:{ruler}<{width}}\n'.format('', ruler=self.ruler, width=80))
+
+                help_topics = self.get_help_topics()
+                for command in cmds:
+                    doc = ''
+                    # Try to get the documentation string
+                    try:
+                        # first see if there's a help function implemented
+                        func = getattr(self, 'help_' + command)
+                    except AttributeError:
+                        # Couldn't find a help function
+                        try:
+                            # Now see if help_summary has been set
+                            doc = getattr(self, self._func_named(command)).help_summary
+                        except AttributeError:
+                            # Last, try to directly ac cess the function's doc-string
+                            doc = getattr(self, self._func_named(command)).__doc__
+                    else:
+                        # we found the help function
+                        result = StringIO()
+                        # try to redirect system stdout
+                        with redirect_stdout(result):
+                            # save our internal stdout
+                            stdout_orig = self.stdout
+                            try:
+                                # redirect our internal stdout
+                                self.stdout = result
+                                func()
+                            finally:
+                                # restore internal stdout
+                                self.stdout = stdout_orig
+                        doc = result.getvalue()
+
+                    # Attempt to locate the first documentation block
+                    doc_block = []
+                    found_first = False
+                    for doc_line in doc.splitlines():
+                        str(doc_line).strip()
+                        if len(doc_line.strip()) > 0:
+                            doc_block.append(doc_line.strip())
+                            found_first = True
+                        else:
+                            if found_first:
+                                break
+
+                    for doc_line in doc_block:
+                        self.stdout.write('{: <{col_width}}{doc}\n'.format(command,
+                                                                           col_width=widest,
+                                                                           doc=doc_line))
+                        command = ''
+                self.stdout.write("\n")
+
     def do_shortcuts(self, _):
         """Lists shortcuts (aliases) available."""
         result = "\n".join('%s: %s' % (sc[0], sc[1]) for sc in sorted(self.shortcuts))
@@ -2848,7 +2971,7 @@ Usage:  Usage: unalias [-a] name [name ...]
         else:
             raise LookupError("Parameter '%s' not supported (type 'show' for list of parameters)." % param)
 
-    set_parser = argparse.ArgumentParser()
+    set_parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
     set_parser.add_argument('-a', '--all', action='store_true', help='display read-only settings as well')
     set_parser.add_argument('-l', '--long', action='store_true', help='describe function of parameter')
     set_parser.add_argument('settable', nargs='*', help='[param_name] [value]')
@@ -3017,6 +3140,8 @@ Usage:  Usage: unalias [-a] name [name ...]
     # noinspection PyBroadException
     def do_py(self, arg):
         """
+        Invoke python command, shell, or script
+
         py <command>: Executes a Python command.
         py: Enters interactive Python mode.
         End with ``Ctrl-D`` (Unix) / ``Ctrl-Z`` (Windows), ``quit()``, '`exit()``.
diff --git a/docs/argument_processing.rst b/docs/argument_processing.rst
index 3ea44657..183dde4e 100644
--- a/docs/argument_processing.rst
+++ b/docs/argument_processing.rst
@@ -160,6 +160,138 @@ Which yields:
    This command can not generate tags with no content, like <br/>
 
 
+Grouping Commands
+=================
+
+By default, the ``help`` command displays::
+
+  Documented commands (type help <topic>):
+  ========================================
+  alias    findleakers  pyscript    sessions             status       vminfo
+  config   help         quit        set                  stop         which
+  connect  history      redeploy    shell                thread_dump
+  deploy   list         resources   shortcuts            unalias
+  edit     load         restart     sslconnectorciphers  undeploy
+  expire   py           serverinfo  start                version
+
+If you have a large number of commands, you can optionally group your commands into categories.
+Here's the output from the example ``help_categories.py``::
+
+  Documented commands (type help <topic>):
+
+  Application Management
+  ======================
+  deploy  findleakers  redeploy  sessions  stop
+  expire  list         restart   start     undeploy
+
+  Connecting
+  ==========
+  connect  which
+
+  Server Information
+  ==================
+  resources  serverinfo  sslconnectorciphers  status  thread_dump  vminfo
+
+  Other
+  =====
+  alias   edit  history  py        quit  shell      unalias
+  config  help  load     pyscript  set   shortcuts  version
+
+
+There are 2 methods of specifying command categories, using the ``@with_category`` decorator or with the
+``categorize()`` function. Once a single command category is detected, the help output switches to a categorized
+mode of display. All commands with an explicit category defined default to the category `Other`.
+
+Using the ``@with_category`` decorator::
+
+  @with_category(CMD_CAT_CONNECTING)
+  def do_which(self, _):
+      """Which command"""
+      self.poutput('Which')
+
+Using the ``categorize()`` function:
+
+    You can call with a single function::
+
+        def do_connect(self, _):
+            """Connect command"""
+            self.poutput('Connect')
+
+        # Tag the above command functions under the category Connecting
+        categorize(do_connect, CMD_CAT_CONNECTING)
+
+    Or with an Iterable container of functions::
+
+        def do_undeploy(self, _):
+            """Undeploy command"""
+            self.poutput('Undeploy')
+
+        def do_stop(self, _):
+            """Stop command"""
+            self.poutput('Stop')
+
+        def do_findleakers(self, _):
+            """Find Leakers command"""
+            self.poutput('Find Leakers')
+
+        # Tag the above command functions under the category Application Management
+        categorize((do_undeploy,
+                    do_stop,
+                    do_findleakers), CMD_CAT_APP_MGMT)
+
+The ``help`` command also has a verbose option (``help -v`` or ``help --verbose``) that combines
+the help categories with per-command Help Messages::
+
+    Documented commands (type help <topic>):
+
+    Application Management
+    ================================================================================
+    deploy              Deploy command
+    expire              Expire command
+    findleakers         Find Leakers command
+    list                List command
+    redeploy            Redeploy command
+    restart             usage: restart [-h] {now,later,sometime,whenever}
+    sessions            Sessions command
+    start               Start command
+    stop                Stop command
+    undeploy            Undeploy command
+
+    Connecting
+    ================================================================================
+    connect             Connect command
+    which               Which command
+
+    Server Information
+    ================================================================================
+    resources              Resources command
+    serverinfo             Server Info command
+    sslconnectorciphers    SSL Connector Ciphers command is an example of a command that contains
+                           multiple lines of help information for the user. Each line of help in a
+                           contiguous set of lines will be printed and aligned in the verbose output
+                           provided with 'help --verbose'
+    status                 Status command
+    thread_dump            Thread Dump command
+    vminfo                 VM Info command
+
+    Other
+    ================================================================================
+    alias               Define or display aliases
+    config              Config command
+    edit                Edit a file in a text editor.
+    help                List available commands with "help" or detailed help with "help cmd".
+    history             usage: history [-h] [-r | -e | -s | -o FILE | -t TRANSCRIPT] [arg]
+    load                Runs commands in script file that is encoded as either ASCII or UTF-8 text.
+    py                  Invoke python command, shell, or script
+    pyscript            Runs a python script file inside the console
+    quit                Exits this application.
+    set                 usage: set [-h] [-a] [-l] [settable [settable ...]]
+    shell               Execute a command as if at the OS prompt.
+    shortcuts           Lists shortcuts (aliases) available.
+    unalias             Unsets aliases
+    version             Version command
+
+
 Receiving an argument list
 ==========================
 
diff --git a/examples/help_categories.py b/examples/help_categories.py
new file mode 100755
index 00000000..e7e3373d
--- /dev/null
+++ b/examples/help_categories.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# coding=utf-8
+"""
+A sample application for tagging categories on commands.
+"""
+
+from cmd2 import Cmd, categorize, __version__, with_argparser, with_category
+import argparse
+
+
+class HelpCategories(Cmd):
+    """ Example cmd2 application. """
+
+    # Command categories
+    CMD_CAT_CONNECTING = 'Connecting'
+    CMD_CAT_APP_MGMT = 'Application Management'
+    CMD_CAT_SERVER_INFO = 'Server Information'
+
+    def __init__(self):
+        # Set use_ipython to True to enable the "ipy" command which embeds and interactive IPython shell
+        Cmd.__init__(self, use_ipython=False)
+
+    def do_connect(self, _):
+        """Connect command"""
+        self.poutput('Connect')
+
+    # Tag the above command functions under the category Connecting
+    categorize(do_connect, CMD_CAT_CONNECTING)
+
+    @with_category(CMD_CAT_CONNECTING)
+    def do_which(self, _):
+        """Which command"""
+        self.poutput('Which')
+
+    def do_list(self, _):
+        """List command"""
+        self.poutput('List')
+
+    def do_deploy(self, _):
+        """Deploy command"""
+        self.poutput('Which')
+
+    def do_start(self, _):
+        """Start command"""
+        self.poutput('Start')
+
+    def do_sessions(self, _):
+        """Sessions command"""
+        self.poutput('Sessions')
+
+    def do_redeploy(self, _):
+        """Redeploy command"""
+        self.poutput('Redeploy')
+
+    restart_parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+    restart_parser.add_argument('when', default='now',
+                                choices=['now', 'later', 'sometime', 'whenever'],
+                                help='Specify when to restart')
+
+    @with_argparser(restart_parser)
+    @with_category(CMD_CAT_APP_MGMT)
+    def do_restart(self, _):
+        """Restart command"""
+        self.poutput('Restart')
+
+    def do_expire(self, _):
+        """Expire command"""
+        self.poutput('Expire')
+
+    def do_undeploy(self, _):
+        """Undeploy command"""
+        self.poutput('Undeploy')
+
+    def do_stop(self, _):
+        """Stop command"""
+        self.poutput('Stop')
+
+    def do_findleakers(self, _):
+        """Find Leakers command"""
+        self.poutput('Find Leakers')
+
+    # Tag the above command functions under the category Application Management
+    categorize((do_list,
+                do_deploy,
+                do_start,
+                do_sessions,
+                do_redeploy,
+                do_expire,
+                do_undeploy,
+                do_stop,
+                do_findleakers), CMD_CAT_APP_MGMT)
+
+    def do_resources(self, _):
+        """Resources command"""
+        self.poutput('Resources')
+
+    def do_status(self, _):
+        """Status command"""
+        self.poutput('Status')
+
+    def do_serverinfo(self, _):
+        """Server Info command"""
+        self.poutput('Server Info')
+
+    def do_thread_dump(self, _):
+        """Thread Dump command"""
+        self.poutput('Thread Dump')
+
+    def do_sslconnectorciphers(self, _):
+        """
+        SSL Connector Ciphers command is an example of a command that contains
+        multiple lines of help information for the user. Each line of help in a
+        contiguous set of lines will be printed and aligned in the verbose output
+        provided with 'help --verbose'
+
+        This is after a blank line and won't de displayed in the verbose help
+        """
+        self.poutput('SSL Connector Ciphers')
+
+    def do_vminfo(self, _):
+        """VM Info command"""
+        self.poutput('VM Info')
+
+    # Tag the above command functions under the category Server Information
+    categorize(do_resources, CMD_CAT_SERVER_INFO)
+    categorize(do_status, CMD_CAT_SERVER_INFO)
+    categorize(do_serverinfo, CMD_CAT_SERVER_INFO)
+    categorize(do_thread_dump, CMD_CAT_SERVER_INFO)
+    categorize(do_sslconnectorciphers, CMD_CAT_SERVER_INFO)
+    categorize(do_vminfo, CMD_CAT_SERVER_INFO)
+
+    # The following command functions don't have the HELP_CATEGORY attribute set
+    # and show up in the 'Other' group
+    def do_config(self, _):
+        """Config command"""
+        self.poutput('Config')
+
+    def do_version(self, _):
+        """Version command"""
+        self.poutput(__version__)
+
+
+if __name__ == '__main__':
+    c = HelpCategories()
+    c.cmdloop()
diff --git a/tests/conftest.py b/tests/conftest.py
index 58ec8ee0..837e7504 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -19,6 +19,23 @@ alias  help     load  pyscript  set    shortcuts
 edit   history  py    quit      shell  unalias
 """
 
+BASE_HELP_VERBOSE = """
+Documented commands (type help <topic>):
+================================================================================
+alias               Define or display aliases
+edit                Edit a file in a text editor.
+help                List available commands with "help" or detailed help with "help cmd".
+history             View, run, edit, and save previously entered commands.
+load                Runs commands in script file that is encoded as either ASCII or UTF-8 text.
+py                  Invoke python command, shell, or script
+pyscript            Runs a python script file inside the console
+quit                Exits this application.
+set                 Sets a settable parameter or shows current settings of parameters.
+shell               Execute a command as if at the OS prompt.
+shortcuts           Lists shortcuts (aliases) available.
+unalias             Unsets aliases
+"""
+
 # Help text for the history command
 HELP_HISTORY = """usage: history [-h] [-r | -e | -s | -o FILE | -t TRANSCRIPT] [arg]
 
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index c675c0b2..878e1605 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -21,7 +21,8 @@ from code import InteractiveConsole
 import six.moves as sm
 
 import cmd2
-from conftest import run_cmd, normalize, BASE_HELP, HELP_HISTORY, SHORTCUTS_TXT, SHOW_TXT, SHOW_LONG, StdOut
+from conftest import run_cmd, normalize, BASE_HELP, BASE_HELP_VERBOSE, \
+    HELP_HISTORY, SHORTCUTS_TXT, SHOW_TXT, SHOW_LONG, StdOut
 
 
 def test_ver():
@@ -38,6 +39,13 @@ def test_base_help(base_app):
     expected = normalize(BASE_HELP)
     assert out == expected
 
+def test_base_help_verbose(base_app):
+    out = run_cmd(base_app, 'help -v')
+    expected = normalize(BASE_HELP_VERBOSE)
+    assert out == expected
+
+    out = run_cmd(base_app, 'help --verbose')
+    assert out == expected
 
 def test_base_help_history(base_app):
     out = run_cmd(base_app, 'help history')
@@ -47,7 +55,7 @@ def test_base_argparse_help(base_app, capsys):
     # Verify that "set -h" gives the same output as "help set" and that it starts in a way that makes sense
     run_cmd(base_app, 'set -h')
     out, err = capsys.readouterr()
-    out1 = out.splitlines()
+    out1 = normalize(str(out))
 
     out2 = run_cmd(base_app, 'help set')
 
@@ -1064,6 +1072,96 @@ def test_help_overridden_method(help_app):
     assert out == expected
 
 
+class HelpCategoriesApp(cmd2.Cmd):
+    """Class for testing custom help_* methods which override docstring help."""
+    def __init__(self, *args, **kwargs):
+        # Need to use this older form of invoking super class constructor to support Python 2.x and Python 3.x
+        cmd2.Cmd.__init__(self, *args, **kwargs)
+
+    @cmd2.with_category('Some Category')
+    def do_diddly(self, arg):
+        """This command does diddly"""
+        pass
+
+    def do_squat(self, arg):
+        """This docstring help will never be shown because the help_squat method overrides it."""
+        pass
+
+    def help_squat(self):
+        self.stdout.write('This command does diddly squat...\n')
+
+    def do_edit(self, arg):
+        """This overrides the edit command and does nothing."""
+        pass
+
+    cmd2.categorize((do_squat, do_edit), 'Custom Category')
+
+    # This command will be in the "undocumented" section of the help menu
+    def do_undoc(self, arg):
+        pass
+
+@pytest.fixture
+def helpcat_app():
+    app = HelpCategoriesApp()
+    app.stdout = StdOut()
+    return app
+
+def test_help_cat_base(helpcat_app):
+    out = run_cmd(helpcat_app, 'help')
+    expected = normalize("""Documented commands (type help <topic>):
+
+Custom Category
+===============
+edit  squat
+
+Some Category
+=============
+diddly
+
+Other
+=====
+alias  help  history  load  py  pyscript  quit  set  shell  shortcuts  unalias
+
+Undocumented commands:
+======================
+undoc
+""")
+    assert out == expected
+
+def test_help_cat_verbose(helpcat_app):
+    out = run_cmd(helpcat_app, 'help --verbose')
+    expected = normalize("""Documented commands (type help <topic>):
+
+Custom Category
+================================================================================
+edit                This overrides the edit command and does nothing.
+squat               This command does diddly squat...
+
+Some Category
+================================================================================
+diddly              This command does diddly
+
+Other
+================================================================================
+alias               Define or display aliases
+help                List available commands with "help" or detailed help with "help cmd".
+history             View, run, edit, and save previously entered commands.
+load                Runs commands in script file that is encoded as either ASCII or UTF-8 text.
+py                  Invoke python command, shell, or script
+pyscript            Runs a python script file inside the console
+quit                Exits this application.
+set                 Sets a settable parameter or shows current settings of parameters.
+shell               Execute a command as if at the OS prompt.
+shortcuts           Lists shortcuts (aliases) available.
+unalias             Unsets aliases
+
+Undocumented commands:
+======================
+undoc
+""")
+    assert out == expected
+
+
 class SelectApp(cmd2.Cmd):
     def do_eat(self, arg):
         """Eat something, with a selection of sauces to choose from."""