Merge pull request #494 from python-cmd2/matches_sorted

Matches sorted

8 files changed, 131 insertions, 32 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2bda144e..fcc394f8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,9 +2,10 @@
 * Bug Fixes
     * Fixed bug where ``preparse`` wasn't getting called
 * Enhancements
-    * Improved implementation of lifecycle hooks to to support a plugin
+    * Improved implementation of lifecycle hooks to support a plugin
       framework, see ``docs/hooks.rst`` for details.
     * New dependency on ``attrs`` third party module
+    * Added ``matches_sorted`` member to support custom sorting of tab-completion matches
 * Deprecations
     * Deprecated the following hook methods, see ``hooks.rst`` for full details:
        * ``cmd2.Cmd.preparse()`` - equivilent functionality available
@@ -14,6 +15,10 @@
        * ``cmd2.Cmd.postparsing_postcmd()`` - equivilent functionality available
          via ``cmd2.Cmd.register_postcmd_hook()``
 
+## 0.8.9 (August TBD, 2018)
+* Bug Fixes
+    * Fixed extra slash that could print when tab completing users on Windows
+
 ## 0.9.3 (July 12, 2018)
 * Bug Fixes
     * Fixed bug when StatementParser ``__init__()`` was called with ``terminators`` equal to ``None``
diff --git a/README.md b/README.md
index a60f0a18..b2eb314c 100755
--- a/README.md
+++ b/README.md
@@ -34,7 +34,6 @@ Main Features
 - Ability to load commands at startup from an initialization script
 - Settable environment parameters
 - Parsing commands with arguments using `argparse`, including support for sub-commands
-- Sub-menu support via the ``AddSubmenu`` decorator
 - Unicode character support
 - Good tab-completion of commands, sub-commands, file system paths, and shell commands
 - Support for Python 3.4+ on Windows, macOS, and Linux
@@ -58,7 +57,7 @@ pip install -U cmd2
 ```
 
 cmd2 works with Python 3.4+ on Windows, macOS, and Linux. It is pure Python code with
-the only 3rd-party dependencies being on [attrs](https://github.com/python-attrs/attrs), 
+the only 3rd-party dependencies being on [attrs](https://github.com/python-attrs/attrs),
 [colorama](https://github.com/tartley/colorama), and [pyperclip](https://github.com/asweigart/pyperclip).
 Windows has an additional dependency on [pyreadline](https://pypi.python.org/pypi/pyreadline). Non-Windows platforms
 have an additional dependency on [wcwidth](https://pypi.python.org/pypi/wcwidth). Finally, Python
diff --git a/cmd2/argparse_completer.py b/cmd2/argparse_completer.py
index 60af25de..1479a6bf 100755
--- a/cmd2/argparse_completer.py
+++ b/cmd2/argparse_completer.py
@@ -492,6 +492,7 @@ class AutoCompleter(object):
 
             self._cmd2_app.completion_header = header
             self._cmd2_app.display_matches = completions_with_desc
+            self._cmd2_app.matches_sorted = True
 
         return completions
 
diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index d34e7161..7273286b 100644
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -369,7 +369,7 @@ class Cmd(cmd.Cmd):
             except AttributeError:
                 pass
 
-		# initialize plugin system
+        # initialize plugin system
         # needs to be done before we call __init__(0)
         self._initialize_plugin_system()
 
@@ -482,11 +482,11 @@ class Cmd(cmd.Cmd):
         # in reset_completion_defaults() and it is up to completer functions to set them before returning results.
         ############################################################################################################
 
-        # If true and a single match is returned to complete(), then a space will be appended
+        # If True and a single match is returned to complete(), then a space will be appended
         # if the match appears at the end of the line
         self.allow_appended_space = True
 
-        # If true and a single match is returned to complete(), then a closing quote
+        # If True and a single match is returned to complete(), then a closing quote
         # will be added if there is an unmatched opening quote
         self.allow_closing_quote = True
 
@@ -504,6 +504,10 @@ class Cmd(cmd.Cmd):
         # quote matches that are completed in a delimited fashion
         self.matches_delimited = False
 
+        # Set to True before returning matches to complete() in cases where matches are sorted with custom ordering.
+        # If False, then complete() will sort the matches alphabetically before they are displayed.
+        self.matches_sorted = False
+
         # Set the pager(s) for use with the ppaged() method for displaying output using a pager
         if sys.platform.startswith('win'):
             self.pager = self.pager_chop = 'more'
@@ -678,6 +682,7 @@ class Cmd(cmd.Cmd):
         self.completion_header = ''
         self.display_matches = []
         self.matches_delimited = False
+        self.matches_sorted = False
 
         if rl_type == RlType.GNU:
             readline.set_completion_display_matches_hook(self._display_matches_gnu_readline)
@@ -994,12 +999,15 @@ class Cmd(cmd.Cmd):
             users = []
 
             # Windows lacks the pwd module so we can't get a list of users.
-            # Instead we will add a slash once the user enters text that
+            # Instead we will return a result once the user enters text that
             # resolves to an existing home directory.
             if sys.platform.startswith('win'):
                 expanded_path = os.path.expanduser(text)
                 if os.path.isdir(expanded_path):
-                    users.append(text + os.path.sep)
+                    user = text
+                    if add_trailing_sep_if_dir:
+                        user += os.path.sep
+                    users.append(user)
             else:
                 import pwd
 
@@ -1083,6 +1091,10 @@ class Cmd(cmd.Cmd):
             self.allow_appended_space = False
             self.allow_closing_quote = False
 
+        # Sort the matches before any trailing slashes are added
+        matches = utils.alphabetical_sort(matches)
+        self.matches_sorted = True
+
         # Build display_matches and add a slash to directories
         for index, cur_match in enumerate(matches):
 
@@ -1446,11 +1458,8 @@ class Cmd(cmd.Cmd):
                 if self.completion_matches:
 
                     # Eliminate duplicates
-                    matches_set = set(self.completion_matches)
-                    self.completion_matches = list(matches_set)
-
-                    display_matches_set = set(self.display_matches)
-                    self.display_matches = list(display_matches_set)
+                    self.completion_matches = utils.remove_duplicates(self.completion_matches)
+                    self.display_matches = utils.remove_duplicates(self.display_matches)
 
                     if not self.display_matches:
                         # Since self.display_matches is empty, set it to self.completion_matches
@@ -1521,10 +1530,11 @@ class Cmd(cmd.Cmd):
 
                 self.completion_matches[0] += str_to_append
 
-            # Otherwise sort matches
-            elif self.completion_matches:
-                self.completion_matches.sort()
-                self.display_matches.sort()
+            # Sort matches alphabetically if they haven't already been sorted
+            if not self.matches_sorted:
+                self.completion_matches = utils.alphabetical_sort(self.completion_matches)
+                self.display_matches = utils.alphabetical_sort(self.display_matches)
+                self.matches_sorted = True
 
         try:
             return self.completion_matches[state]
@@ -2270,7 +2280,7 @@ Usage:  Usage: unalias [-a] name [name ...]
 
         else:
             # Get rid of duplicates
-            arglist = list(set(arglist))
+            arglist = utils.remove_duplicates(arglist)
 
             for cur_arg in arglist:
                 if cur_arg in self.aliases:
@@ -2315,12 +2325,10 @@ Usage:  Usage: unalias [-a] name [name ...]
         """Show a list of commands which help can be displayed for.
         """
         # Get a sorted list of help topics
-        help_topics = self.get_help_topics()
-        help_topics.sort()
+        help_topics = utils.alphabetical_sort(self.get_help_topics())
 
         # Get a sorted list of visible command names
-        visible_commands = self.get_visible_commands()
-        visible_commands.sort()
+        visible_commands = utils.alphabetical_sort(self.get_visible_commands())
 
         cmds_doc = []
         cmds_undoc = []
diff --git a/cmd2/utils.py b/cmd2/utils.py
index d03e7f6f..02956f6b 100644
--- a/cmd2/utils.py
+++ b/cmd2/utils.py
@@ -5,6 +5,7 @@
 import collections
 import os
 from typing import Any, List, Optional, Union
+import unicodedata
 
 from . import constants
 
@@ -110,7 +111,7 @@ def which(editor: str) -> Optional[str]:
 
 
 def is_text_file(file_path: str) -> bool:
-    """Returns if a file contains only ASCII or UTF-8 encoded text
+    """Returns if a file contains only ASCII or UTF-8 encoded text.
 
     :param file_path: path to the file being checked
     :return: True if the file is a text file, False if it is binary.
@@ -144,3 +145,34 @@ def is_text_file(file_path: str) -> bool:
             pass
 
     return valid_text_file
+
+
+def remove_duplicates(list_to_prune: List) -> List:
+    """Removes duplicates from a list while preserving order of the items.
+
+    :param list_to_prune: the list being pruned of duplicates
+    :return: The pruned list
+    """
+    temp_dict = collections.OrderedDict()
+    for item in list_to_prune:
+        temp_dict[item] = None
+
+    return list(temp_dict.keys())
+
+
+def norm_fold(astr: str) -> str:
+    """Normalize and casefold Unicode strings for saner comparisons.
+
+    :param astr: input unicode string
+    :return: a normalized and case-folded version of the input string
+    """
+    return unicodedata.normalize('NFC', astr).casefold()
+
+
+def alphabetical_sort(list_to_sort: List[str]) -> List[str]:
+    """Sorts a list of strings alphabetically.
+
+    :param list_to_sort: the list being sorted
+    :return: the sorted list
+    """
+    return sorted(list_to_sort, key=norm_fold)
diff --git a/examples/tab_autocompletion.py b/examples/tab_autocompletion.py
index 342cfff5..38972358 100755
--- a/examples/tab_autocompletion.py
+++ b/examples/tab_autocompletion.py
@@ -38,7 +38,7 @@ class TabCompleteExample(cmd2.Cmd):
     static_list_directors = ['J. J. Abrams', 'Irvin Kershner', 'George Lucas', 'Richard Marquand',
                              'Rian Johnson', 'Gareth Edwards']
     USER_MOVIE_LIBRARY = ['ROGUE1', 'SW_EP04', 'SW_EP05']
-    MOVIE_DATABASE_IDS = ['SW_EP01', 'SW_EP02', 'SW_EP03', 'ROGUE1', 'SW_EP04',
+    MOVIE_DATABASE_IDS = ['SW_EP1', 'SW_EP02', 'SW_EP03', 'ROGUE1', 'SW_EP04',
                           'SW_EP05', 'SW_EP06', 'SW_EP07', 'SW_EP08', 'SW_EP09']
     MOVIE_DATABASE = {'SW_EP04': {'title': 'Star Wars: Episode IV - A New Hope',
                                   'rating': 'PG',
@@ -52,13 +52,13 @@ class TabCompleteExample(cmd2.Cmd):
                                   'actor': ['Mark Hamill', 'Harrison Ford', 'Carrie Fisher',
                                             'Alec Guinness', 'Peter Mayhew', 'Anthony Daniels']
                                   },
-                      'SW_EP06': {'title': 'Star Wars: Episode IV - A New Hope',
+                      'SW_EP06': {'title': 'Star Wars: Episode VI - Return of the Jedi',
                                   'rating': 'PG',
                                   'director': ['Richard Marquand'],
                                   'actor': ['Mark Hamill', 'Harrison Ford', 'Carrie Fisher',
                                             'Alec Guinness', 'Peter Mayhew', 'Anthony Daniels']
                                   },
-                      'SW_EP01': {'title': 'Star Wars: Episode I - The Phantom Menace',
+                      'SW_EP1': {'title': 'Star Wars: Episode I - The Phantom Menace',
                                   'rating': 'PG',
                                   'director': ['George Lucas'],
                                   'actor': ['Liam Neeson', 'Ewan McGregor', 'Natalie Portman', 'Jake Lloyd']
@@ -113,8 +113,10 @@ class TabCompleteExample(cmd2.Cmd):
         """Demonstrates showing tabular hinting of tab completion information"""
         completions_with_desc = []
 
-        for movie_id, movie_entry in self.MOVIE_DATABASE.items():
-            completions_with_desc.append(argparse_completer.CompletionItem(movie_id, movie_entry['title']))
+        for movie_id in self.MOVIE_DATABASE_IDS:
+            if movie_id in self.MOVIE_DATABASE:
+                movie_entry = self.MOVIE_DATABASE[movie_id]
+                completions_with_desc.append(argparse_completer.CompletionItem(movie_id, movie_entry['title']))
 
         return completions_with_desc
 
diff --git a/tests/test_completion.py b/tests/test_completion.py
index 2faa4a08..00a120cc 100644
--- a/tests/test_completion.py
+++ b/tests/test_completion.py
@@ -14,6 +14,7 @@ import sys
 
 import pytest
 import cmd2
+from cmd2 import utils
 from .conftest import complete_tester, StdOut
 from examples.subcommands import SubcommandsExample
 
@@ -251,7 +252,7 @@ def test_path_completion_multiple(cmd2_app, request):
     endidx = len(line)
     begidx = endidx - len(text)
 
-    matches = sorted(cmd2_app.path_complete(text, line, begidx, endidx))
+    matches = cmd2_app.path_complete(text, line, begidx, endidx)
     expected = [text + 'cript.py', text + 'cript.txt', text + 'cripts' + os.path.sep]
     assert matches == expected
 
@@ -408,9 +409,8 @@ def test_delimiter_completion(cmd2_app):
     cmd2_app.delimiter_complete(text, line, begidx, endidx, delimited_strs, '/')
 
     # Remove duplicates from display_matches and sort it. This is typically done in complete().
-    display_set = set(cmd2_app.display_matches)
-    display_list = list(display_set)
-    display_list.sort()
+    display_list = utils.remove_duplicates(cmd2_app.display_matches)
+    display_list = utils.alphabetical_sort(display_list)
 
     assert display_list == ['other user', 'user']
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 00000000..61fd8373
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,52 @@
+# coding=utf-8
+"""
+Unit testing for cmd2/utils.py module.
+
+Copyright 2018 Todd Leonhardt <todd.leonhardt@gmail.com>
+Released under MIT license, see LICENSE file
+"""
+from colorama import Fore
+import cmd2.utils as cu
+
+HELLO_WORLD = 'Hello, world!'
+
+
+def test_strip_ansi():
+    base_str = HELLO_WORLD
+    ansi_str =  Fore.GREEN + base_str + Fore.RESET
+    assert base_str != ansi_str
+    assert base_str == cu.strip_ansi(ansi_str)
+
+def test_strip_quotes_no_quotes():
+    base_str = HELLO_WORLD
+    stripped = cu.strip_quotes(base_str)
+    assert base_str == stripped
+
+def test_strip_quotes_with_quotes():
+    base_str = '"' + HELLO_WORLD + '"'
+    stripped = cu.strip_quotes(base_str)
+    assert stripped == HELLO_WORLD
+
+def test_remove_duplicates_no_duplicates():
+    no_dups = [5, 4, 3, 2, 1]
+    assert cu.remove_duplicates(no_dups) == no_dups
+
+def test_remove_duplicates_with_duplicates():
+    duplicates = [1, 1, 2, 3, 9, 9, 7, 8]
+    assert cu.remove_duplicates(duplicates) == [1, 2, 3, 9, 7, 8]
+
+def test_unicode_normalization():
+    s1 = 'café'
+    s2 = 'cafe\u0301'
+    assert s1 != s2
+    assert cu.norm_fold(s1) == cu.norm_fold(s2)
+
+def test_unicode_casefold():
+    micro = 'µ'
+    micro_cf = micro.casefold()
+    assert micro != micro_cf
+    assert cu.norm_fold(micro) == cu.norm_fold(micro_cf)
+
+def test_alphabetical_sort():
+    my_list = ['café', 'µ', 'A' , 'micro', 'unity', 'cafeteria']
+    assert cu.alphabetical_sort(my_list) == ['A', 'cafeteria', 'café', 'micro', 'unity', 'µ']