diff options
author | kotfu <jared@kotfu.net> | 2018-05-24 19:07:45 -0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-24 19:07:45 -0600 |
commit | cad21a60fa92ebe4a7c177142d273f9f7497967b (patch) | |
tree | 3cfa590653af8b7570bd4073384467d2ca736fa0 | |
parent | 5d64ebee348aeffb02fc385f903c9af431e3721b (diff) | |
parent | 190fecb34ac91e25f64615f378d6d59ef6d77de8 (diff) | |
download | cmd2-git-cad21a60fa92ebe4a7c177142d273f9f7497967b.tar.gz |
Merge pull request #413 from python-cmd2/speedup_import
Defer certain imports
31 files changed, 630 insertions, 480 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index f9627194..6bb29e48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ * ``cmd2.redirector`` is no longer supported. Output redirection can only be done with '>' or '>>' * Python 2 no longer supported * ``cmd2`` now supports Python 3.4+ +* Known Issues + * Some developers have noted very slow performance when importing the ``cmd2`` module. The issue + it intermittant, and investigation of the root cause is ongoing. ## 0.8.5 (April 15, 2018) * Bug Fixes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4ebcbef..24bd835e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,9 +21,9 @@ Working on your first Pull Request? You can learn how from this *free* series [H - [Prerequisites](#prerequisites) - [Forking The Project](#forking-the-project) - [Create A Branch](#create-a-branch) -- [Setup Linting](#setup-linting) - [Setup for cmd2 development](#setup-for-cmd2-development) - [Make Changes](#make-changes) +- [Static Code Analysis](#static-code-analysis) - [Run The Test Suite](#run-the-test-suite) - [Squash Your Commits](#squash-your-commits) - [Creating A Pull Request](#creating-a-pull-request) @@ -33,6 +33,7 @@ Working on your first Pull Request? You can learn how from this *free* series [H - [Next Steps](#next-steps) - [Other resources](#other-resources) - [Advice](#advice) +- [Developing in an IDE](#developing-in-an-ide) ### Prerequisites @@ -175,57 +176,44 @@ $ git push origin [name_of_your_new_branch] ##### If you need more help with branching, take a look at _[this](https://github.com/Kunena/Kunena-Forum/wiki/Create-a-new-branch-with-git-and-manage-branches)_. -### Setup Linting - -You should have some sort of [PEP8](https://www.python.org/dev/peps/pep-0008/)-based linting running in your editor or IDE or at the command-line before you commit code. [pylint](https://www.pylint.org) is a good Python linter which can be run at the command-line but also can integrate with many IDEs and editors. - -> Please do not ignore any linting errors in code you write or modify, as they are meant to **help** you and to ensure a clean and simple code base. Don't worry about linting errors in code you don't touch though - cleaning up the legacy code is a work in progress. - ### Setup for cmd2 development -Once you have cmd2 cloned, before you start any cmd2 application, you first need to install all of the dependencies: - -```bash -# Install cmd2 prerequisites -pip install -U pyperclip - -# Install prerequisites for running cmd2 unit tests -pip install -U pytest - -# Install prerequisites for building cmd2 documentation -pip install -U sphinx sphinx-rtd-theme - -# Install optional prerequisites for doing code coverage analysis -pip install -U pytest-cov -``` - -For doing cmd2 development, you actually do NOT want to have cmd2 installed as a Python package. +For doing cmd2 development, you actually do NOT want to have cmd2 installed as a Python package. So if you have previously installed cmd2, make sure to uninstall it: ```bash pip uninstall cmd2 ``` -Then you should modify your PYTHONPATH environment variable to include the directory you have cloned the cmd2 repository to. -Add a line similar to the following to your .bashrc, .bashprofile, or to your Windows environment variables: - +Assuming you cloned the repository to `~/src/cmd2`: ```bash -# Use cmd2 Python module from GitHub clone when it isn't installed -export PYTHONPATH=$PYTHONPATH:~/src/cmd2 +$ cd ~/src/cmd2 +$ pip install -e . ``` +will install cmd2 in [editable mode](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs). +Changes to the source code are immediately available when the python interpreter +imports `cmd2`, there is no need to re-install the module after every change. This +command will also install all of the runtime dependencies for `cmd2`. -Where `~src/cmd2` is replaced by the directory you cloned your fork of the cmd2 repo to. +Next you should install all the modules used for development of `cmd2`: +```bash +$ cd ~/src/cmd2 +$ pip install -e .[dev] +``` -Now navigate to your terminal to the directory you cloned your fork of the cmd2 repo to and -try running the example to make sure everything is working: +This will install `pytest` and `tox` for running unit tests, `pylint` for +static code analysis, and `sphinx` for building the documentation. +Now you can check if everything is installed and working: ```bash cd ~src/cmd2 python examples/example.py ``` -If the example app loads, you should see a prompt that says "(Cmd)". You can type `help` to get help or `quit` to quit. -If you see that, then congratulations – you're all set. Otherwise, refer to the cmd2 [Installation Instructions](https://cmd2.readthedocs.io/en/latest/install.html#installing). There also might be an error in the console -of your Bash / Terminal / Command Line that will help identify the problem. +If the example app loads, you should see a prompt that says "(Cmd)". You can +type `help` to get help or `quit` to quit. If you see that, then congratulations +– you're all set. Otherwise, refer to the cmd2 [Installation Instructions](https://cmd2.readthedocs.io/en/latest/install.html#installing). +There also might be an error in the console of your Bash / Terminal / Command Line +that will help identify the problem. ### Make Changes This bit is up to you! @@ -246,17 +234,20 @@ make clean html ``` In order to see the changes, use your web browser of choice to open `<cmd2>/docs/_build/html/index.html`. +### Static Code Analysis + +You should have some sort of [PEP8](https://www.python.org/dev/peps/pep-0008/)-based linting running in your editor or IDE or at the command-line before you commit code. [pylint](https://www.pylint.org) is a good Python linter which can be run at the command-line but also can integrate with many IDEs and editors. + +> Please do not ignore any linting errors in code you write or modify, as they are meant to **help** you and to ensure a clean and simple code base. Don't worry about linting errors in code you don't touch though - cleaning up the legacy code is a work in progress. + ### Run The Test Suite When you're ready to share your code, run the test suite: - ```shell cd <cmd2> py.test ``` - and ensure all tests pass. - #### Measuring code coverage Code coverage can be measured as follows: @@ -381,7 +372,7 @@ how to do it. 7. Creating the PR causes our continuous integration (CI) systems to automatically run all of the unit tests on all supported OSes and all supported versions of Python. You should watch your PR - to make sure that all unit tests pass on Both TravisCI (Linux) and AppVeyor (Windows). + to make sure that all unit tests pass on Both TravisCI (Linux) and AppVeyor (Windows). 8. If any unit tests fail, you should look at the details and fix the failures. You can then push the fix to the same branch in your fork and the PR will automatically get updated and the CI system @@ -395,7 +386,7 @@ integration (CI) providers to automatically run all of the unit tests on multipl 1. If your changes can merge without conflicts and all unit tests pass for all OSes and supported versions of Python, then your pull request (PR) will have a big green checkbox which says something like "All Checks Passed" next to it. -If this is not the case, there will be a link you can click on to get details regarding what the problem is. +If this is not the case, there will be a link you can click on to get details regarding what the problem is. It is your responsibility to make sure all unit tests are passing. Generally a Maintainer will not QA a pull request unless it can merge without conflicts and all unit tests pass on all supported platforms. @@ -460,9 +451,9 @@ Here is some advice regarding what makes a good pull request (PR) from the persp - Code coverage of the unit tests matters, try not to decrease it - Think twice before adding dependencies to 3rd party libraries (outside of the Python standard library) because it could affect a lot of users -### Developing and Debugging in an IDE +### Developing in an IDE -We recommend using [Visual Studio Code](https://code.visualstudio.com) with the [Python extension](https://code.visualstudio.com/docs/languages/python) and it's [Integrated Terminal](https://code.visualstudio.com/docs/python/debugging) debugger for debugging since it has +We recommend using [Visual Studio Code](https://code.visualstudio.com) with the [Python extension](https://code.visualstudio.com/docs/languages/python) and it's [Integrated Terminal](https://code.visualstudio.com/docs/python/debugging) debugger for debugging since it has excellent support for debugging console applications. [PyCharm](https://www.jetbrains.com/pycharm/) is also quite good and has very nice [Code Inspection](https://www.jetbrains.com/help/pycharm/code-inspection.html) capabilities. diff --git a/cmd2/__init__.py b/cmd2/__init__.py index bf6c047f..617d643b 100644 --- a/cmd2/__init__.py +++ b/cmd2/__init__.py @@ -1,2 +1,4 @@ # # -*- coding: utf-8 -*- +from .cmd2 import __version__, Cmd, CmdResult, Statement, EmptyStatement, categorize +from .cmd2 import with_argument_list, with_argparser, with_argparser_and_unknown_args, with_category diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index a2d67def..f480b3ae 100755 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -22,30 +22,24 @@ is used in place of `print`. Git repository on GitHub at https://github.com/python-cmd2/cmd2 """ +# This module has many imports, quite a few of which are only +# infrequently utilized. To reduce the initial overhead of +# import this module, many of these imports are lazy-loaded +# i.e. we only import the module when we use it +# For example, we don't import the 'traceback' module +# until the perror() function is called and the debug +# setting is True import argparse -import atexit import cmd -import codecs import collections from colorama import Fore -import copy -import datetime -import functools import glob -import io -from io import StringIO import os import platform import re import shlex -import signal -import subprocess import sys -import tempfile -import traceback -from typing import Callable, List, Optional, Union, Tuple -import unittest -from code import InteractiveConsole +from typing import Callable, List, Union, Tuple import pyperclip @@ -146,16 +140,6 @@ def categorize(func: Union[Callable, Iterable], category: str) -> None: else: setattr(func, HELP_CATEGORY, category) - -def _which(editor: str) -> Optional[str]: - try: - editor_path = subprocess.check_output(['which', editor], stderr=subprocess.STDOUT).strip() - editor_path = editor_path.decode() - except subprocess.CalledProcessError: - editor_path = None - return editor_path - - def parse_quoted_string(cmdline: str) -> List[str]: """Parse a quoted string into a list of arguments.""" if isinstance(cmdline, list): @@ -185,6 +169,8 @@ def with_argument_list(func: Callable) -> Callable: method. Default passes a string of whatever the user typed. With this decorator, the decorated method will receive a list of arguments parsed from user input using shlex.split().""" + import functools + @functools.wraps(func) def cmd_wrapper(self, cmdline): lexed_arglist = parse_quoted_string(cmdline) @@ -201,6 +187,8 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser) -> Calla :param argparser: argparse.ArgumentParser - given instance of ArgumentParser :return: function that gets passed parsed args and a list of unknown args """ + import functools + # noinspection PyProtectedMember def arg_decorator(func: Callable): @functools.wraps(func) @@ -241,6 +229,7 @@ def with_argparser(argparser: argparse.ArgumentParser) -> Callable: :param argparser: argparse.ArgumentParser - given instance of ArgumentParser :return: function that gets passed parsed args """ + import functools # noinspection PyProtectedMember def arg_decorator(func: Callable): @@ -361,7 +350,7 @@ class Cmd(cmd.Cmd): else: # Favor command-line editors first so we don't leave the terminal to edit for editor in ['vim', 'vi', 'emacs', 'nano', 'pico', 'gedit', 'kate', 'subl', 'geany', 'atom']: - if _which(editor): + if utils.which(editor): break feedback_to_output = False # Do not include nonessentials in >, | output by default (things like timing) locals_in_py = False @@ -410,6 +399,7 @@ class Cmd(cmd.Cmd): readline.set_history_length(persistent_history_length) except FileNotFoundError: pass + import atexit atexit.register(readline.write_history_file, persistent_history_file) # Call super class constructor @@ -555,6 +545,7 @@ class Cmd(cmd.Cmd): :return: """ if self.debug: + import traceback traceback.print_exc() if exception_type is None: @@ -586,6 +577,7 @@ class Cmd(cmd.Cmd): :param msg: str - message to print to current stdout - anything convertible to a str with '{}'.format() is OK :param end: str - string appended after the end of the message if not already present, default a newline """ + import subprocess if msg is not None and msg != '': try: msg_str = '{}'.format(msg) @@ -684,6 +676,7 @@ class Cmd(cmd.Cmd): On Failure Both items are None """ + import copy unclosed_quote = '' quotes_to_try = copy.copy(constants.QUOTES) @@ -1289,6 +1282,7 @@ class Cmd(cmd.Cmd): :param text: str - the current word that user is typing :param state: int - non-negative integer """ + import functools if state == 0 and rl_type != RlType.NONE: unclosed_quote = '' self.set_completion_defaults() @@ -1423,6 +1417,7 @@ class Cmd(cmd.Cmd): # Since self.display_matches is empty, set it to self.completion_matches # before we alter them. That way the suggestions will reflect how we parsed # the token being completed and not how readline did. + import copy self.display_matches = copy.copy(self.completion_matches) # Check if we need to add an opening quote @@ -1594,7 +1589,7 @@ class Cmd(cmd.Cmd): def preloop(self): """"Hook method executed once when the cmdloop() method is called.""" - + import signal # Register a default SIGINT signal handler for Ctrl+C signal.signal(signal.SIGINT, self.sigint_handler) @@ -1658,6 +1653,7 @@ class Cmd(cmd.Cmd): if not sys.platform.startswith('win'): # Fix those annoying problems that occur with terminal programs like "less" when you pipe to them if self.stdin.isatty(): + import subprocess proc = subprocess.Popen(shlex.split('stty sane')) proc.communicate() return stop @@ -1682,6 +1678,7 @@ class Cmd(cmd.Cmd): :param line: str - line of text read from input :return: bool - True if cmdloop() should exit, False otherwise """ + import datetime stop = False try: statement = self._complete_statement(line) @@ -1801,6 +1798,9 @@ class Cmd(cmd.Cmd): :param statement: Statement - a parsed statement from the user """ + import io + import subprocess + if statement.pipe_to: self.kept_state = Statekeeper(self, ('stdout',)) @@ -1829,6 +1829,7 @@ class Cmd(cmd.Cmd): # Re-raise the exception raise ex elif statement.output: + import tempfile if (not statement.output_to) and (not can_clip): raise EnvironmentError("Cannot redirect to paste buffer; install 'pyperclip' and re-run to enable") self.kept_state = Statekeeper(self, ('stdout',)) @@ -2260,6 +2261,8 @@ Usage: Usage: unalias [-a] name [name ...] def _print_topics(self, header, cmds, verbose): """Customized version of print_topics that can switch between verbose or traditional output""" + import io + if cmds: if not verbose: self.print_topics(header, cmds, 15, 80) @@ -2294,7 +2297,7 @@ Usage: Usage: unalias [-a] name [name ...] doc = getattr(self, self._func_named(command)).__doc__ else: # we found the help function - result = StringIO() + result = io.StringIO() # try to redirect system stdout with redirect_stdout(result): # save our internal stdout @@ -2445,7 +2448,7 @@ Usage: Usage: unalias [-a] name [name ...] if (val[0] == val[-1]) and val[0] in ("'", '"'): val = val[1:-1] else: - val = cast(current_val, val) + val = utils.cast(current_val, val) setattr(self, param_name, val) self.poutput('%s - was: %s\nnow: %s\n' % (param_name, current_val, val)) if current_val != val: @@ -2465,6 +2468,7 @@ Usage: Usage: unalias [-a] name [name ...] Usage: shell <command> [arguments]""" + import subprocess try: # Use non-POSIX parsing to keep the quotes around the tokens tokens = shlex.split(command, posix=False) @@ -2551,6 +2555,7 @@ Usage: Usage: unalias [-a] name [name ...] self.pystate['self'] = self localvars = self.pystate + from code import InteractiveConsole interp = InteractiveConsole(locals=localvars) interp.runcode('import sys, os;sys.path.insert(0, os.getcwd())') @@ -2696,6 +2701,7 @@ a..b, a:b, a:, ..b items by indices (inclusive) if runme: self.onecmd_plus_hooks(runme) elif args.edit: + import tempfile fd, fname = tempfile.mkstemp(suffix='.txt', text=True) with os.fdopen(fd, 'w') as fobj: for command in history: @@ -2730,6 +2736,8 @@ a..b, a:b, a:, ..b items by indices (inclusive) """Generate a transcript file from a given history of commands.""" # Save the current echo state, and turn it off. We inject commands into the # output using a different mechanism + import io + saved_echo = self.echo self.echo = False @@ -2880,7 +2888,7 @@ Script should contain one command per line, just like command would be typed in return # Make sure the file is ASCII or UTF-8 encoded text - if not self.is_text_file(expanded_path): + if not utils.is_text_file(expanded_path): self.perror('{} is not an ASCII or UTF-8 encoded text file'.format(expanded_path), traceback_war=False) return @@ -2901,40 +2909,6 @@ Script should contain one command per line, just like command would be typed in index_dict = {1: self.path_complete} return self.index_based_complete(text, line, begidx, endidx, index_dict) - @staticmethod - def is_text_file(file_path): - """ - Returns if a file contains only ASCII or UTF-8 encoded text - :param file_path: path to the file being checked - """ - expanded_path = os.path.abspath(os.path.expanduser(file_path.strip())) - valid_text_file = False - - # Check if the file is ASCII - try: - with codecs.open(expanded_path, encoding='ascii', errors='strict') as f: - # Make sure the file has at least one line of text - # noinspection PyUnusedLocal - if sum(1 for line in f) > 0: - valid_text_file = True - except IOError: # pragma: no cover - pass - except UnicodeDecodeError: - # The file is not ASCII. Check if it is UTF-8. - try: - with codecs.open(expanded_path, encoding='utf-8', errors='strict') as f: - # Make sure the file has at least one line of text - # noinspection PyUnusedLocal - if sum(1 for line in f) > 0: - valid_text_file = True - except IOError: # pragma: no cover - pass - except UnicodeDecodeError: - # Not UTF-8 - pass - - return valid_text_file - def run_transcript_tests(self, callargs): """Runs transcript tests for provided file(s). @@ -2943,6 +2917,8 @@ Script should contain one command per line, just like command would be typed in :param callargs: List[str] - list of transcript test file names """ + import unittest + from .transcript import Cmd2TestCase class TestMyAppCase(Cmd2TestCase): cmdapp = self @@ -3130,36 +3106,6 @@ class History(list): return [itm for itm in self if isin(itm)] -def cast(current, new): - """Tries to force a new value into the same type as the current when trying to set the value for a parameter. - - :param current: current value for the parameter, type varies - :param new: str - new value - :return: new value with same type as current, or the current value if there was an error casting - """ - typ = type(current) - if typ == bool: - try: - return bool(int(new)) - except (ValueError, TypeError): - pass - try: - new = new.lower() - except AttributeError: - pass - if (new == 'on') or (new[0] in ('y', 't')): - return True - if (new == 'off') or (new[0] in ('n', 'f')): - return False - else: - try: - return typ(new) - except (ValueError, TypeError): - pass - print("Problem setting parameter (now %s) to %s; incorrect type?" % (current, new)) - return current - - class Statekeeper(object): """Class used to save and restore state during load and py commands as well as when redirecting output or pipes.""" def __init__(self, obj, attribs): @@ -3185,232 +3131,7 @@ class Statekeeper(object): setattr(self.obj, attrib, getattr(self, attrib)) -class OutputTrap(object): - """Instantiate an OutputTrap to divert/capture ALL stdout output. For use in transcript testing.""" - - def __init__(self): - self.contents = '' - - def write(self, txt): - """Add text to the internal contents. - - :param txt: str - """ - self.contents += txt - - def read(self): - """Read from the internal contents and then clear them out. - - :return: str - text from the internal contents - """ - result = self.contents - self.contents = '' - return result - - -class Cmd2TestCase(unittest.TestCase): - """Subclass this, setting CmdApp, to make a unittest.TestCase class - that will execute the commands in a transcript file and expect the results shown. - See example.py""" - cmdapp = None - - def fetchTranscripts(self): - self.transcripts = {} - for fileset in self.cmdapp.testfiles: - for fname in glob.glob(fileset): - tfile = open(fname) - self.transcripts[fname] = iter(tfile.readlines()) - tfile.close() - if not len(self.transcripts): - raise Exception("No test files found - nothing to test.") - - def setUp(self): - if self.cmdapp: - self.fetchTranscripts() - - # Trap stdout - self._orig_stdout = self.cmdapp.stdout - self.cmdapp.stdout = OutputTrap() - - def runTest(self): # was testall - if self.cmdapp: - its = sorted(self.transcripts.items()) - for (fname, transcript) in its: - self._test_transcript(fname, transcript) - - def _test_transcript(self, fname, transcript): - line_num = 0 - finished = False - line = utils.strip_ansi(next(transcript)) - line_num += 1 - while not finished: - # Scroll forward to where actual commands begin - while not line.startswith(self.cmdapp.visible_prompt): - try: - line = utils.strip_ansi(next(transcript)) - except StopIteration: - finished = True - break - line_num += 1 - command = [line[len(self.cmdapp.visible_prompt):]] - line = next(transcript) - # Read the entirety of a multi-line command - while line.startswith(self.cmdapp.continuation_prompt): - command.append(line[len(self.cmdapp.continuation_prompt):]) - try: - line = next(transcript) - except StopIteration: - raise (StopIteration, - 'Transcript broke off while reading command beginning at line {} with\n{}'.format(line_num, - command[0]) - ) - line_num += 1 - command = ''.join(command) - # Send the command into the application and capture the resulting output - # TODO: Should we get the return value and act if stop == True? - self.cmdapp.onecmd_plus_hooks(command) - result = self.cmdapp.stdout.read() - # Read the expected result from transcript - if utils.strip_ansi(line).startswith(self.cmdapp.visible_prompt): - message = '\nFile {}, line {}\nCommand was:\n{}\nExpected: (nothing)\nGot:\n{}\n'.format( - fname, line_num, command, result) - self.assert_(not (result.strip()), message) - continue - expected = [] - while not utils.strip_ansi(line).startswith(self.cmdapp.visible_prompt): - expected.append(line) - try: - line = next(transcript) - except StopIteration: - finished = True - break - line_num += 1 - expected = ''.join(expected) - - # transform the expected text into a valid regular expression - expected = self._transform_transcript_expected(expected) - message = '\nFile {}, line {}\nCommand was:\n{}\nExpected:\n{}\nGot:\n{}\n'.format( - fname, line_num, command, expected, result) - self.assertTrue(re.match(expected, result, re.MULTILINE | re.DOTALL), message) - - def _transform_transcript_expected(self, s): - """parse the string with slashed regexes into a valid regex - - Given a string like: - - Match a 10 digit phone number: /\d{3}-\d{3}-\d{4}/ - - Turn it into a valid regular expression which matches the literal text - of the string and the regular expression. We have to remove the slashes - because they differentiate between plain text and a regular expression. - Unless the slashes are escaped, in which case they are interpreted as - plain text, or there is only one slash, which is treated as plain text - also. - - Check the tests in tests/test_transcript.py to see all the edge - cases. - """ - regex = '' - start = 0 - - while True: - (regex, first_slash_pos, start) = self._escaped_find(regex, s, start, False) - if first_slash_pos == -1: - # no more slashes, add the rest of the string and bail - regex += re.escape(s[start:]) - break - else: - # there is a slash, add everything we have found so far - # add stuff before the first slash as plain text - regex += re.escape(s[start:first_slash_pos]) - start = first_slash_pos+1 - # and go find the next one - (regex, second_slash_pos, start) = self._escaped_find(regex, s, start, True) - if second_slash_pos > 0: - # add everything between the slashes (but not the slashes) - # as a regular expression - regex += s[start:second_slash_pos] - # and change where we start looking for slashed on the - # turn through the loop - start = second_slash_pos + 1 - else: - # No closing slash, we have to add the first slash, - # and the rest of the text - regex += re.escape(s[start-1:]) - break - return regex - - @staticmethod - def _escaped_find(regex, s, start, in_regex): - """ - Find the next slash in {s} after {start} that is not preceded by a backslash. - - If we find an escaped slash, add everything up to and including it to regex, - updating {start}. {start} therefore serves two purposes, tells us where to start - looking for the next thing, and also tells us where in {s} we have already - added things to {regex} - - {in_regex} specifies whether we are currently searching in a regex, we behave - differently if we are or if we aren't. - """ - - while True: - pos = s.find('/', start) - if pos == -1: - # no match, return to caller - break - elif pos == 0: - # slash at the beginning of the string, so it can't be - # escaped. We found it. - break - else: - # check if the slash is preceeded by a backslash - if s[pos-1:pos] == '\\': - # it is. - if in_regex: - # add everything up to the backslash as a - # regular expression - regex += s[start:pos-1] - # skip the backslash, and add the slash - regex += s[pos] - else: - # add everything up to the backslash as escaped - # plain text - regex += re.escape(s[start:pos-1]) - # and then add the slash as escaped - # plain text - regex += re.escape(s[pos]) - # update start to show we have handled everything - # before it - start = pos+1 - # and continue to look - else: - # slash is not escaped, this is what we are looking for - break - return regex, pos, start - - def tearDown(self): - if self.cmdapp: - # Restore stdout - self.cmdapp.stdout = self._orig_stdout - - -def namedtuple_with_two_defaults(typename, field_names, default_values=('', '')): - """Wrapper around namedtuple which lets you treat the last value as optional. - - :param typename: str - type name for the Named tuple - :param field_names: List[str] or space-separated string of field names - :param default_values: (optional) 2-element tuple containing the default values for last 2 parameters in named tuple - Defaults to an empty string for both of them - :return: namedtuple type - """ - T = collections.namedtuple(typename, field_names) - # noinspection PyUnresolvedReferences - T.__new__.__defaults__ = default_values - return T - - -class CmdResult(namedtuple_with_two_defaults('CmdResult', ['out', 'err', 'war'])): +class CmdResult(utils.namedtuple_with_two_defaults('CmdResult', ['out', 'err', 'war'])): """Derive a class to store results from a named tuple so we can tweak dunder methods for convenience. This is provided as a convenience and an example for one possible way for end users to store results in @@ -3430,5 +3151,3 @@ class CmdResult(namedtuple_with_two_defaults('CmdResult', ['out', 'err', 'war']) def __bool__(self): """If err is an empty string, treat the result as a success; otherwise treat it as a failure.""" return not self.err - - diff --git a/cmd2/transcript.py b/cmd2/transcript.py new file mode 100644 index 00000000..8a9837a6 --- /dev/null +++ b/cmd2/transcript.py @@ -0,0 +1,228 @@ +# +# -*- coding: utf-8 -*- +"""Machinery for running and validating transcripts. + +If the user wants to run a transcript (see docs/transcript.rst), +we need a mechanism to run each command in the transcript as +a unit test, comparing the expected output to the actual output. + +This file contains the classess necessary to make that work. These +classes are used in cmd2.py::run_transcript_tests() +""" +import re +import glob +import unittest + +from . import utils + +class Cmd2TestCase(unittest.TestCase): + """A unittest class used for transcript testing. + + Subclass this, setting CmdApp, to make a unittest.TestCase class + that will execute the commands in a transcript file and expect the + results shown. + + See example.py + """ + cmdapp = None + + def fetchTranscripts(self): + self.transcripts = {} + for fileset in self.cmdapp.testfiles: + for fname in glob.glob(fileset): + tfile = open(fname) + self.transcripts[fname] = iter(tfile.readlines()) + tfile.close() + if not len(self.transcripts): + raise Exception("No test files found - nothing to test.") + + def setUp(self): + if self.cmdapp: + self.fetchTranscripts() + + # Trap stdout + self._orig_stdout = self.cmdapp.stdout + self.cmdapp.stdout = OutputTrap() + + def runTest(self): # was testall + if self.cmdapp: + its = sorted(self.transcripts.items()) + for (fname, transcript) in its: + self._test_transcript(fname, transcript) + + def _test_transcript(self, fname, transcript): + line_num = 0 + finished = False + line = utils.strip_ansi(next(transcript)) + line_num += 1 + while not finished: + # Scroll forward to where actual commands begin + while not line.startswith(self.cmdapp.visible_prompt): + try: + line = utils.strip_ansi(next(transcript)) + except StopIteration: + finished = True + break + line_num += 1 + command = [line[len(self.cmdapp.visible_prompt):]] + line = next(transcript) + # Read the entirety of a multi-line command + while line.startswith(self.cmdapp.continuation_prompt): + command.append(line[len(self.cmdapp.continuation_prompt):]) + try: + line = next(transcript) + except StopIteration as exc: + msg = 'Transcript broke off while reading command beginning at line {} with\n{}'.format(line_num, command[0]) + raise StopIteration(msg) from exc + line_num += 1 + command = ''.join(command) + # Send the command into the application and capture the resulting output + # TODO: Should we get the return value and act if stop == True? + self.cmdapp.onecmd_plus_hooks(command) + result = self.cmdapp.stdout.read() + # Read the expected result from transcript + if utils.strip_ansi(line).startswith(self.cmdapp.visible_prompt): + message = '\nFile {}, line {}\nCommand was:\n{}\nExpected: (nothing)\nGot:\n{}\n'.format( + fname, line_num, command, result) + self.assert_(not (result.strip()), message) + continue + expected = [] + while not utils.strip_ansi(line).startswith(self.cmdapp.visible_prompt): + expected.append(line) + try: + line = next(transcript) + except StopIteration: + finished = True + break + line_num += 1 + expected = ''.join(expected) + + # transform the expected text into a valid regular expression + expected = self._transform_transcript_expected(expected) + message = '\nFile {}, line {}\nCommand was:\n{}\nExpected:\n{}\nGot:\n{}\n'.format( + fname, line_num, command, expected, result) + self.assertTrue(re.match(expected, result, re.MULTILINE | re.DOTALL), message) + + def _transform_transcript_expected(self, s): + """Parse the string with slashed regexes into a valid regex. + + Given a string like: + + Match a 10 digit phone number: /\d{3}-\d{3}-\d{4}/ + + Turn it into a valid regular expression which matches the literal text + of the string and the regular expression. We have to remove the slashes + because they differentiate between plain text and a regular expression. + Unless the slashes are escaped, in which case they are interpreted as + plain text, or there is only one slash, which is treated as plain text + also. + + Check the tests in tests/test_transcript.py to see all the edge + cases. + """ + regex = '' + start = 0 + + while True: + (regex, first_slash_pos, start) = self._escaped_find(regex, s, start, False) + if first_slash_pos == -1: + # no more slashes, add the rest of the string and bail + regex += re.escape(s[start:]) + break + else: + # there is a slash, add everything we have found so far + # add stuff before the first slash as plain text + regex += re.escape(s[start:first_slash_pos]) + start = first_slash_pos+1 + # and go find the next one + (regex, second_slash_pos, start) = self._escaped_find(regex, s, start, True) + if second_slash_pos > 0: + # add everything between the slashes (but not the slashes) + # as a regular expression + regex += s[start:second_slash_pos] + # and change where we start looking for slashed on the + # turn through the loop + start = second_slash_pos + 1 + else: + # No closing slash, we have to add the first slash, + # and the rest of the text + regex += re.escape(s[start-1:]) + break + return regex + + @staticmethod + def _escaped_find(regex, s, start, in_regex): + """Find the next slash in {s} after {start} that is not preceded by a backslash. + + If we find an escaped slash, add everything up to and including it to regex, + updating {start}. {start} therefore serves two purposes, tells us where to start + looking for the next thing, and also tells us where in {s} we have already + added things to {regex} + + {in_regex} specifies whether we are currently searching in a regex, we behave + differently if we are or if we aren't. + """ + + while True: + pos = s.find('/', start) + if pos == -1: + # no match, return to caller + break + elif pos == 0: + # slash at the beginning of the string, so it can't be + # escaped. We found it. + break + else: + # check if the slash is preceeded by a backslash + if s[pos-1:pos] == '\\': + # it is. + if in_regex: + # add everything up to the backslash as a + # regular expression + regex += s[start:pos-1] + # skip the backslash, and add the slash + regex += s[pos] + else: + # add everything up to the backslash as escaped + # plain text + regex += re.escape(s[start:pos-1]) + # and then add the slash as escaped + # plain text + regex += re.escape(s[pos]) + # update start to show we have handled everything + # before it + start = pos+1 + # and continue to look + else: + # slash is not escaped, this is what we are looking for + break + return regex, pos, start + + def tearDown(self): + if self.cmdapp: + # Restore stdout + self.cmdapp.stdout = self._orig_stdout + +class OutputTrap(object): + """Instantiate an OutputTrap to divert/capture ALL stdout output. + For use in transcript testing. + """ + + def __init__(self): + self.contents = '' + + def write(self, txt): + """Add text to the internal contents. + + :param txt: str + """ + self.contents += txt + + def read(self): + """Read from the internal contents and then clear them out. + + :return: str - text from the internal contents + """ + result = self.contents + self.contents = '' + return result diff --git a/cmd2/utils.py b/cmd2/utils.py index dbe39213..07969ff1 100644 --- a/cmd2/utils.py +++ b/cmd2/utils.py @@ -3,6 +3,9 @@ """Shared utility functions""" import collections +import os +from typing import Optional + from . import constants def strip_ansi(text: str) -> str: @@ -55,3 +58,98 @@ def namedtuple_with_defaults(typename, field_names, default_values=()): T.__new__.__defaults__ = tuple(prototype) return T +def namedtuple_with_two_defaults(typename, field_names, default_values=('', '')): + """Wrapper around namedtuple which lets you treat the last value as optional. + + :param typename: str - type name for the Named tuple + :param field_names: List[str] or space-separated string of field names + :param default_values: (optional) 2-element tuple containing the default values for last 2 parameters in named tuple + Defaults to an empty string for both of them + :return: namedtuple type + """ + T = collections.namedtuple(typename, field_names) + # noinspection PyUnresolvedReferences + T.__new__.__defaults__ = default_values + return T + +def cast(current, new): + """Tries to force a new value into the same type as the current when trying to set the value for a parameter. + + :param current: current value for the parameter, type varies + :param new: str - new value + :return: new value with same type as current, or the current value if there was an error casting + """ + typ = type(current) + if typ == bool: + try: + return bool(int(new)) + except (ValueError, TypeError): + pass + try: + new = new.lower() + except AttributeError: + pass + if (new == 'on') or (new[0] in ('y', 't')): + return True + if (new == 'off') or (new[0] in ('n', 'f')): + return False + else: + try: + return typ(new) + except (ValueError, TypeError): + pass + print("Problem setting parameter (now %s) to %s; incorrect type?" % (current, new)) + return current + +def which(editor: str) -> Optional[str]: + """Find the full path of a given editor. + + Return the full path of the given editor, or None if the editor can + not be found. + + :param editor: filename of the editor to check, ie 'notepad.exe' or 'vi' + :return: a full path or None + """ + import subprocess + try: + editor_path = subprocess.check_output(['which', editor], stderr=subprocess.STDOUT).strip() + editor_path = editor_path.decode() + except subprocess.CalledProcessError: + editor_path = None + return editor_path + +def is_text_file(file_path): + """Returns if a file contains only ASCII or UTF-8 encoded text + + :param file_path: path to the file being checked + :return: True if the file is a text file, False if it is binary. + """ + import codecs + + expanded_path = os.path.abspath(os.path.expanduser(file_path.strip())) + valid_text_file = False + + # Check if the file is ASCII + try: + with codecs.open(expanded_path, encoding='ascii', errors='strict') as f: + # Make sure the file has at least one line of text + # noinspection PyUnusedLocal + if sum(1 for line in f) > 0: + valid_text_file = True + except IOError: # pragma: no cover + pass + except UnicodeDecodeError: + # The file is not ASCII. Check if it is UTF-8. + try: + with codecs.open(expanded_path, encoding='utf-8', errors='strict') as f: + # Make sure the file has at least one line of text + # noinspection PyUnusedLocal + if sum(1 for line in f) > 0: + valid_text_file = True + except IOError: # pragma: no cover + pass + except UnicodeDecodeError: + # Not UTF-8 + pass + + return valid_text_file diff --git a/examples/alias_startup.py b/examples/alias_startup.py index 7ccfa6e5..4a061e87 100755 --- a/examples/alias_startup.py +++ b/examples/alias_startup.py @@ -5,8 +5,7 @@ 2) How to load an initialization script at startup """ -from cmd2 import cmd2 - +import cmd2 class AliasAndStartup(cmd2.Cmd): """ Example cmd2 application where we create commands that just print the arguments they are called with.""" diff --git a/examples/arg_print.py b/examples/arg_print.py index b2f0fcda..4f0ca709 100755 --- a/examples/arg_print.py +++ b/examples/arg_print.py @@ -2,18 +2,17 @@ # coding=utf-8 """A simple example demonstrating the following: 1) How arguments and options get parsed and passed to commands - 2) How to change what syntax get parsed as a comment and stripped from the arguments + 2) How to change what syntax get parsed as a comment and stripped from + the arguments -This is intended to serve as a live demonstration so that developers can experiment with and understand how command -and argument parsing is intended to work. +This is intended to serve as a live demonstration so that developers can +experiment with and understand how command and argument parsing work. It also serves as an example of how to create command aliases (shortcuts). """ import argparse -from cmd2 import cmd2 -from cmd2.cmd2 import with_argument_list, with_argparser, with_argparser_and_unknown_args - +import cmd2 class ArgumentAndOptionPrinter(cmd2.Cmd): """ Example cmd2 application where we create commands that just print the arguments they are called with.""" @@ -27,11 +26,14 @@ class ArgumentAndOptionPrinter(cmd2.Cmd): # NOTE: It is critical that the super class __init__ method be called AFTER updating certain parameters which # are not settable at runtime. This includes the shortcuts, multiline_commands, etc. - def do_aprint(self, arg): + def do_aprint(self, statement): """Print the argument string this basic command is called with.""" - self.poutput('aprint was called with argument: {!r}'.format(arg)) + self.poutput('aprint was called with argument: {!r}'.format(statement)) + self.poutput('statement.raw = {!r}'.format(statement.raw)) + self.poutput('statement.argv = {!r}'.format(statement.argv)) + self.poutput('statement.command = {!r}'.format(statement.command)) - @with_argument_list + @cmd2.with_argument_list def do_lprint(self, arglist): """Print the argument list this basic command is called with.""" self.poutput('lprint was called with the following list of arguments: {!r}'.format(arglist)) @@ -42,7 +44,7 @@ class ArgumentAndOptionPrinter(cmd2.Cmd): oprint_parser.add_argument('-r', '--repeat', type=int, help='output [n] times') oprint_parser.add_argument('words', nargs='+', help='words to print') - @with_argparser(oprint_parser) + @cmd2.with_argparser(oprint_parser) def do_oprint(self, args): """Print the options and argument list this options command was called with.""" self.poutput('oprint was called with the following\n\toptions: {!r}'.format(args)) @@ -51,13 +53,12 @@ class ArgumentAndOptionPrinter(cmd2.Cmd): pprint_parser.add_argument('-p', '--piglatin', action='store_true', help='atinLay') pprint_parser.add_argument('-s', '--shout', action='store_true', help='N00B EMULATION MODE') pprint_parser.add_argument('-r', '--repeat', type=int, help='output [n] times') - @with_argparser_and_unknown_args(pprint_parser) + @cmd2.with_argparser_and_unknown_args(pprint_parser) def do_pprint(self, args, unknown): """Print the options and argument list this options command was called with.""" self.poutput('oprint was called with the following\n\toptions: {!r}\n\targuments: {}'.format(args, unknown)) - if __name__ == '__main__': app = ArgumentAndOptionPrinter() app.cmdloop() diff --git a/examples/argparse_example.py b/examples/argparse_example.py index 6e5dcf35..236e2af4 100755 --- a/examples/argparse_example.py +++ b/examples/argparse_example.py @@ -14,10 +14,10 @@ verifying that the output produced matches the transcript. import argparse import sys -from cmd2.cmd2 import Cmd, with_argparser, with_argument_list +import cmd2 -class CmdLineApp(Cmd): +class CmdLineApp(cmd2.Cmd): """ Example cmd2 application. """ def __init__(self, ip_addr=None, port=None, transcript_files=None): self.multiline_commands = ['orate'] @@ -46,7 +46,7 @@ class CmdLineApp(Cmd): speak_parser.add_argument('-r', '--repeat', type=int, help='output [n] times') speak_parser.add_argument('words', nargs='+', help='words to say') - @with_argparser(speak_parser) + @cmd2.with_argparser(speak_parser) def do_speak(self, args): """Repeats what you tell me to.""" words = [] @@ -67,13 +67,13 @@ class CmdLineApp(Cmd): tag_parser.add_argument('tag', help='tag') tag_parser.add_argument('content', nargs='+', help='content to surround with tag') - @with_argparser(tag_parser) + @cmd2.with_argparser(tag_parser) def do_tag(self, args): """create a html tag""" self.poutput('<{0}>{1}</{0}>'.format(args.tag, ' '.join(args.content))) - @with_argument_list + @cmd2.with_argument_list def do_tagg(self, arglist): """verion of creating an html tag using arglist instead of argparser""" if len(arglist) >= 2: diff --git a/examples/environment.py b/examples/environment.py index af452e4e..c45ce71c 100755 --- a/examples/environment.py +++ b/examples/environment.py @@ -4,10 +4,10 @@ A sample application for cmd2 demonstrating customized environment parameters """ -from cmd2.cmd2 import Cmd +import cmd2 -class EnvironmentApp(Cmd): +class EnvironmentApp(cmd2.Cmd): """ Example cmd2 application. """ degrees_c = 22 diff --git a/examples/event_loops.py b/examples/event_loops.py index a76c5d91..53d3ca2b 100755 --- a/examples/event_loops.py +++ b/examples/event_loops.py @@ -6,7 +6,7 @@ This is an example of how to use cmd2 in a way so that cmd2 doesn't own the inne This opens up the possibility of registering cmd2 input with event loops, like asyncio, without occupying the main loop. """ -from cmd2 import cmd2 +import cmd2 class Cmd2EventBased(cmd2.Cmd): diff --git a/examples/example.py b/examples/example.py index f07b9c74..264abd84 100755 --- a/examples/example.py +++ b/examples/example.py @@ -14,10 +14,10 @@ the transcript. import random import argparse -from cmd2.cmd2 import Cmd, with_argparser +import cmd2 -class CmdLineApp(Cmd): +class CmdLineApp(cmd2.Cmd): """ Example cmd2 application. """ # Setting this true makes it run a shell command if a cmd2/cmd command doesn't exist @@ -43,7 +43,7 @@ class CmdLineApp(Cmd): speak_parser.add_argument('-r', '--repeat', type=int, help='output [n] times') speak_parser.add_argument('words', nargs='+', help='words to say') - @with_argparser(speak_parser) + @cmd2.with_argparser(speak_parser) def do_speak(self, args): """Repeats what you tell me to.""" words = [] @@ -65,7 +65,7 @@ class CmdLineApp(Cmd): mumble_parser.add_argument('-r', '--repeat', type=int, help='how many times to repeat') mumble_parser.add_argument('words', nargs='+', help='words to say') - @with_argparser(mumble_parser) + @cmd2.with_argparser(mumble_parser) def do_mumble(self, args): """Mumbles what you tell me to.""" repetitions = args.repeat or 1 diff --git a/examples/help_categories.py b/examples/help_categories.py index dcfbd31f..b4a2c977 100755 --- a/examples/help_categories.py +++ b/examples/help_categories.py @@ -6,10 +6,9 @@ A sample application for tagging categories on commands. import argparse -from cmd2.cmd2 import Cmd, categorize, __version__, with_argparser, with_category +import cmd2 - -class HelpCategories(Cmd): +class HelpCategories(cmd2.Cmd): """ Example cmd2 application. """ # Command categories @@ -26,9 +25,9 @@ class HelpCategories(Cmd): self.poutput('Connect') # Tag the above command functions under the category Connecting - categorize(do_connect, CMD_CAT_CONNECTING) + cmd2.categorize(do_connect, CMD_CAT_CONNECTING) - @with_category(CMD_CAT_CONNECTING) + @cmd2.with_category(CMD_CAT_CONNECTING) def do_which(self, _): """Which command""" self.poutput('Which') @@ -58,8 +57,8 @@ class HelpCategories(Cmd): choices=['now', 'later', 'sometime', 'whenever'], help='Specify when to restart') - @with_argparser(restart_parser) - @with_category(CMD_CAT_APP_MGMT) + @cmd2.with_argparser(restart_parser) + @cmd2.with_category(CMD_CAT_APP_MGMT) def do_restart(self, _): """Restart command""" self.poutput('Restart') @@ -81,7 +80,7 @@ class HelpCategories(Cmd): self.poutput('Find Leakers') # Tag the above command functions under the category Application Management - categorize((do_list, + cmd2.categorize((do_list, do_deploy, do_start, do_sessions, @@ -123,12 +122,12 @@ class HelpCategories(Cmd): self.poutput('VM Info') # Tag the above command functions under the category Server Information - categorize(do_resources, CMD_CAT_SERVER_INFO) - categorize(do_status, CMD_CAT_SERVER_INFO) - categorize(do_serverinfo, CMD_CAT_SERVER_INFO) - categorize(do_thread_dump, CMD_CAT_SERVER_INFO) - categorize(do_sslconnectorciphers, CMD_CAT_SERVER_INFO) - categorize(do_vminfo, CMD_CAT_SERVER_INFO) + cmd2.categorize(do_resources, CMD_CAT_SERVER_INFO) + cmd2.categorize(do_status, CMD_CAT_SERVER_INFO) + cmd2.categorize(do_serverinfo, CMD_CAT_SERVER_INFO) + cmd2.categorize(do_thread_dump, CMD_CAT_SERVER_INFO) + cmd2.categorize(do_sslconnectorciphers, CMD_CAT_SERVER_INFO) + cmd2.categorize(do_vminfo, CMD_CAT_SERVER_INFO) # The following command functions don't have the HELP_CATEGORY attribute set # and show up in the 'Other' group @@ -138,7 +137,7 @@ class HelpCategories(Cmd): def do_version(self, _): """Version command""" - self.poutput(__version__) + self.poutput(cmd2.__version__) if __name__ == '__main__': diff --git a/examples/paged_output.py b/examples/paged_output.py index 9396f04e..c56dcb89 100755 --- a/examples/paged_output.py +++ b/examples/paged_output.py @@ -3,8 +3,7 @@ """A simple example demonstrating the using paged output via the ppaged() method. """ -from cmd2 import cmd2 -from cmd2.cmd2 import with_argument_list +import cmd2 class PagedOutput(cmd2.Cmd): @@ -13,7 +12,7 @@ class PagedOutput(cmd2.Cmd): def __init__(self): super().__init__() - @with_argument_list + @cmd2.with_argument_list def do_page_file(self, args): """Read in a text file and display its output in a pager.""" if not args: diff --git a/examples/persistent_history.py b/examples/persistent_history.py index 251dbd67..61e26b9c 100755 --- a/examples/persistent_history.py +++ b/examples/persistent_history.py @@ -5,7 +5,7 @@ This will allow end users of your cmd2-based application to use the arrow keys and Ctrl+r in a manner which persists across invocations of your cmd2 application. This can make it much easier for them to use your application. """ -from cmd2 import cmd2 +import cmd2 class Cmd2PersistentHistory(cmd2.Cmd): diff --git a/examples/pirate.py b/examples/pirate.py index f6f4c629..34906a9f 100755 --- a/examples/pirate.py +++ b/examples/pirate.py @@ -7,15 +7,16 @@ presented as part of her PyCon 2010 talk. It demonstrates many features of cmd2. """ import argparse -from cmd2.cmd2 import Cmd, with_argparser +import cmd2 -class Pirate(Cmd): + +class Pirate(cmd2.Cmd): """A piratical example cmd2 application involving looting and drinking.""" def __init__(self): self.default_to_shell = True self.multiline_commands = ['sing'] - self.terminators = Cmd.terminators + ['...'] + self.terminators = self.terminators + ['...'] self.songcolor = 'blue' # Add stuff to settable and/or shortcuts before calling base class initializer @@ -74,7 +75,7 @@ class Pirate(Cmd): yo_parser.add_argument('-c', '--commas', action='store_true', help='Intersperse commas') yo_parser.add_argument('beverage', help='beverage to drink with the chant') - @with_argparser(yo_parser) + @cmd2.with_argparser(yo_parser) def do_yo(self, args): """Compose a yo-ho-ho type chant with flexible options.""" chant = ['yo'] + ['ho'] * args.ho diff --git a/examples/python_scripting.py b/examples/python_scripting.py index 865cf052..7e2cf345 100755 --- a/examples/python_scripting.py +++ b/examples/python_scripting.py @@ -17,7 +17,7 @@ This application and the "scripts/conditional.py" script serve as an example for import argparse import os -from cmd2 import cmd2 +import cmd2 class CmdLineApp(cmd2.Cmd): diff --git a/examples/remove_unused.py b/examples/remove_unused.py index dfe0a055..8a567123 100755 --- a/examples/remove_unused.py +++ b/examples/remove_unused.py @@ -9,7 +9,7 @@ name, they just won't clutter the help menu. Commands can also be removed entirely by using Python's "del". """ -from cmd2 import cmd2 +import cmd2 class RemoveUnusedBuiltinCommands(cmd2.Cmd): diff --git a/examples/subcommands.py b/examples/subcommands.py index 55be7711..356c2e09 100755 --- a/examples/subcommands.py +++ b/examples/subcommands.py @@ -52,8 +52,7 @@ except ImportError: # Intentionally below the bash completion code to reduce tab completion lag -from cmd2 import cmd2 -from cmd2.cmd2 import with_argparser +import cmd2 class SubcommandsExample(cmd2.Cmd): @@ -83,7 +82,7 @@ class SubcommandsExample(cmd2.Cmd): parser_bar.set_defaults(func=base_bar) parser_sport.set_defaults(func=base_sport) - @with_argparser(base_parser) + @cmd2.with_argparser(base_parser) def do_base(self, args): """Base command help""" func = getattr(args, 'func', None) diff --git a/examples/tab_autocompletion.py b/examples/tab_autocompletion.py index adfe9702..d1726841 100755 --- a/examples/tab_autocompletion.py +++ b/examples/tab_autocompletion.py @@ -10,8 +10,8 @@ import argparse import itertools from typing import List -from cmd2 import cmd2, argparse_completer -from cmd2.cmd2 import with_argparser, with_category +import cmd2 +from cmd2 import argparse_completer actors = ['Mark Hamill', 'Harrison Ford', 'Carrie Fisher', 'Alec Guinness', 'Peter Mayhew', 'Anthony Daniels', 'Adam Driver', 'Daisy Ridley', 'John Boyega', 'Oscar Isaac', @@ -122,8 +122,8 @@ class TabCompleteExample(cmd2.Cmd): '\tsingle value - maximum duration\n' '\t[a, b] - duration range') - @with_category(CAT_AUTOCOMPLETE) - @with_argparser(suggest_parser) + @cmd2.with_category(CAT_AUTOCOMPLETE) + @cmd2.with_argparser(suggest_parser) def do_suggest(self, args) -> None: """Suggest command demonstrates argparse customizations @@ -147,8 +147,8 @@ class TabCompleteExample(cmd2.Cmd): '\tsingle value - maximum duration\n' '\t[a, b] - duration range') - @with_category(CAT_AUTOCOMPLETE) - @with_argparser(suggest_parser_hybrid) + @cmd2.with_category(CAT_AUTOCOMPLETE) + @cmd2.with_argparser(suggest_parser_hybrid) def do_hybrid_suggest(self, args): if not args.type: self.do_help('orig_suggest') @@ -165,8 +165,8 @@ class TabCompleteExample(cmd2.Cmd): '\tsingle value - maximum duration\n' '\t[a, b] - duration range') - @with_argparser(suggest_parser_orig) - @with_category(CAT_AUTOCOMPLETE) + @cmd2.with_argparser(suggest_parser_orig) + @cmd2.with_category(CAT_AUTOCOMPLETE) def do_orig_suggest(self, args) -> None: if not args.type: self.do_help('orig_suggest') @@ -261,8 +261,8 @@ class TabCompleteExample(cmd2.Cmd): vid_shows_list_parser = vid_shows_commands_subparsers.add_parser('list') - @with_category(CAT_AUTOCOMPLETE) - @with_argparser(video_parser) + @cmd2.with_category(CAT_AUTOCOMPLETE) + @cmd2.with_argparser(video_parser) def do_video(self, args): """Video management command demonstrates multiple layers of subcommands being handled by AutoCompleter""" func = getattr(args, 'func', None) @@ -339,8 +339,8 @@ class TabCompleteExample(cmd2.Cmd): shows_list_parser = shows_commands_subparsers.add_parser('list') - @with_category(CAT_AUTOCOMPLETE) - @with_argparser(media_parser) + @cmd2.with_category(CAT_AUTOCOMPLETE) + @cmd2.with_argparser(media_parser) def do_media(self, args): """Media management command demonstrates multiple layers of subcommands being handled by AutoCompleter""" func = getattr(args, 'func', None) @@ -440,8 +440,8 @@ class TabCompleteExample(cmd2.Cmd): return self._filter_library(text, line, begidx, endidx, all_episodes, user_eps) return [] - @with_category(CAT_AUTOCOMPLETE) - @with_argparser(library_parser) + @cmd2.with_category(CAT_AUTOCOMPLETE) + @cmd2.with_argparser(library_parser) def do_library(self, args): """Media management command demonstrates multiple layers of subcommands being handled by AutoCompleter""" func = getattr(args, 'func', None) diff --git a/examples/tab_completion.py b/examples/tab_completion.py index 30fa283d..2ec7ff70 100755 --- a/examples/tab_completion.py +++ b/examples/tab_completion.py @@ -4,8 +4,7 @@ """ import argparse -from cmd2 import cmd2 -from cmd2.cmd2 import with_argparser, with_argument_list +import cmd2 # List of strings used with flag and index based completion functions food_item_strs = ['Pizza', 'Ham', 'Ham Sandwich', 'Potato'] @@ -24,7 +23,7 @@ class TabCompleteExample(cmd2.Cmd): add_item_group.add_argument('-s', '--sport', help='Adds sport item') add_item_group.add_argument('-o', '--other', help='Adds other item') - @with_argparser(add_item_parser) + @cmd2.with_argparser(add_item_parser) def do_add_item(self, args): """Add item command help""" if args.food: @@ -57,7 +56,7 @@ class TabCompleteExample(cmd2.Cmd): return self.flag_based_complete(text, line, begidx, endidx, flag_dict=flag_dict) - @with_argument_list + @cmd2.with_argument_list def do_list_item(self, args): """List item command help""" self.poutput("You listed {}".format(args)) diff --git a/examples/table_display.py b/examples/table_display.py index 5d168408..2e6ea804 100755 --- a/examples/table_display.py +++ b/examples/table_display.py @@ -12,7 +12,7 @@ WARNING: This example requires the tabulate module. """ import functools -from cmd2 import cmd2 +import cmd2 import tabulate # Format to use with tabulate module when displaying tables diff --git a/mtime.sh b/mtime.sh new file mode 100755 index 00000000..1cb5f8dc --- /dev/null +++ b/mtime.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +TMPFILE=`mktemp /tmp/mtime.XXXXXX` || exit 1 + +for x in {1..100} +do + gtime -f "real %e user %U sys %S" -a -o $TMPFILE "$@" + #tail -1 $TMPFILE +done + +awk '{ et += $2; ut += $4; st += $6; count++ } END { printf "%d iterations\n", count ; printf "average: real %.3f user %.3f sys %.3f\n", et/count, ut/count, st/count }' $TMPFILE + +rm $TMPFILE + @@ -67,10 +67,13 @@ EXTRAS_REQUIRE = { ":sys_platform!='win32'": ['wcwidth'], # Python 3.4 and earlier require contextlib2 for temporarily redirecting stderr and stdout ":python_version<'3.5'": ['contextlib2', 'typing'], + # development only dependencies + # install with 'pip install -e .[dev]' + 'dev': [ + 'pytest', 'pytest-cov', 'tox', 'pylint', 'sphinx', 'sphinx-rtd-theme', + ] } -TESTS_REQUIRE = ['pytest', 'pytest-xdist'] - setup( name="cmd2", version=VERSION, @@ -87,5 +90,4 @@ setup( python_requires='>=3.4', install_requires=INSTALL_REQUIRES, extras_require=EXTRAS_REQUIRE, - tests_require=TESTS_REQUIRE, ) diff --git a/speedup_import.md b/speedup_import.md new file mode 100644 index 00000000..c49f1e86 --- /dev/null +++ b/speedup_import.md @@ -0,0 +1,99 @@ +# Speedup Import + +## Assumptions + +I created a simple script to run a command 20 times and calculate +the average clock time for each run of the command. This script requires +some unix tools, including the gnu flavor of the `time` command. This script +can is called `mtime.sh` and is included in this branch. + +These tests were all run on my 2015 MacBook Pro with a 3.1 GHz Intel Core i7 +and 16GB of memory. + + +## Baseline measurement + +First let's see how long it takes to start up python. The longish path here +ensures we aren't measuring the time it takes the pyenv shims to run: +``` +$./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "" +100 iterations +average: real 0.028 user 0.020 sys 0.000 +``` + + +## Initial measurement + +From commit fbbfe256, which has `__init.py__` importing `cmd2.cmd2.Cmd` +and a bunch of other stuff, we get: +``` +$ ./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2" +100 iterations +average: real 0.140 user 0.100 sys 0.030 +``` + +From the baseline and this initial measurement, we infer it takes ~110 ms +to import the `cmd2` module. + + +## Defer unittest + +In commit 8bc2c37a we defer the import of `unittest` until we need it to +test a transcript. +``` +$./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2" +100 iterations +average: real 0.131 user 0.091 sys 0.030 +``` + + +## Defer InteractiveConsole from code + +In commit 6e49661f we defer the import of `InteractiveConsole` until the user +wants to run the `py` command. +``` +$ ./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2" +100 iterations +average: real 0.131 user 0.090 sys 0.030 +``` + +## Defer atexit, codes, signal, tempfile, copy + +In commit a479fa94 we defer 5 imports: atexit, codecs, signal, tempfile, and copy. +``` +$ ./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2"100 iterations +average: real 0.120 user 0.081 sys 0.021 +``` + +## Defer datetime, functools, io, subprocess, traceback + +In commit d9ca07a9 we defer 5 more imports: datetime, functools, io, subprocess, traceback. +``` +$ ./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2" +100 iterations +average: real 0.115 user 0.080 sys 0.020 +``` + +## extract AddSubmenu to its own file + +In commit ccfdf0f9 we extract AddSubmenu() to it's own file, so it is not +imported or processed by default. +``` +$ ./mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2" +100 iterations +average: real 0.117 user 0.081 sys 0.021 +``` + +## Progress Update + +Python takes ~30ms to start up and do nothing. When we began we estimated it took +~110ms to import cmd2. We are now down to about ~90ms, which is approximately a +20% improvement. + +## Move more functions into utils + +Commit fc495a42 moves a few functions from `cmd2.py` into `utils.py`. +``` +$ ~/bin/mtime.sh ~/.pyenv/versions/cmd2-3.6/bin/python -c "import cmd2" +100 iterations +average: real 0.119 user 0.081 sys 0.021 diff --git a/tests/conftest.py b/tests/conftest.py index 562ca4fa..90d45bd9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ import sys from pytest import fixture from unittest import mock -from cmd2 import cmd2 +import cmd2 # Prefer statically linked gnureadline if available (for macOS compatibility due to issues with libedit) try: diff --git a/tests/test_argparse.py b/tests/test_argparse.py index f1a2b357..469cbe76 100644 --- a/tests/test_argparse.py +++ b/tests/test_argparse.py @@ -5,7 +5,7 @@ Cmd2 testing for argument parsing import argparse import pytest -from cmd2 import cmd2 +import cmd2 from unittest import mock from .conftest import run_cmd, StdOut diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py index 6e4a5a3e..662983f9 100644 --- a/tests/test_cmd2.py +++ b/tests/test_cmd2.py @@ -8,9 +8,9 @@ Released under MIT license, see LICENSE file import argparse import builtins from code import InteractiveConsole +import io import os import sys -import io import tempfile import pytest @@ -21,7 +21,8 @@ try: except ImportError: from unittest import mock -from cmd2 import cmd2 +import cmd2 +from cmd2 import utils from .conftest import run_cmd, normalize, BASE_HELP, BASE_HELP_VERBOSE, \ HELP_HISTORY, SHORTCUTS_TXT, SHOW_TXT, SHOW_LONG, StdOut @@ -109,44 +110,40 @@ def test_base_show_readonly(base_app): def test_cast(): - cast = cmd2.cast - # Boolean - assert cast(True, True) == True - assert cast(True, False) == False - assert cast(True, 0) == False - assert cast(True, 1) == True - assert cast(True, 'on') == True - assert cast(True, 'off') == False - assert cast(True, 'ON') == True - assert cast(True, 'OFF') == False - assert cast(True, 'y') == True - assert cast(True, 'n') == False - assert cast(True, 't') == True - assert cast(True, 'f') == False + assert utils.cast(True, True) == True + assert utils.cast(True, False) == False + assert utils.cast(True, 0) == False + assert utils.cast(True, 1) == True + assert utils.cast(True, 'on') == True + assert utils.cast(True, 'off') == False + assert utils.cast(True, 'ON') == True + assert utils.cast(True, 'OFF') == False + assert utils.cast(True, 'y') == True + assert utils.cast(True, 'n') == False + assert utils.cast(True, 't') == True + assert utils.cast(True, 'f') == False # Non-boolean same type - assert cast(1, 5) == 5 - assert cast(3.4, 2.7) == 2.7 - assert cast('foo', 'bar') == 'bar' - assert cast([1,2], [3,4]) == [3,4] + assert utils.cast(1, 5) == 5 + assert utils.cast(3.4, 2.7) == 2.7 + assert utils.cast('foo', 'bar') == 'bar' + assert utils.cast([1,2], [3,4]) == [3,4] def test_cast_problems(capsys): - cast = cmd2.cast - expected = 'Problem setting parameter (now {}) to {}; incorrect type?\n' # Boolean current, with new value not convertible to bool current = True new = [True, True] - assert cast(current, new) == current + assert utils.cast(current, new) == current out, err = capsys.readouterr() assert out == expected.format(current, new) # Non-boolean current, with new value not convertible to current type current = 1 new = 'octopus' - assert cast(current, new) == current + assert utils.cast(current, new) == current out, err = capsys.readouterr() assert out == expected.format(current, new) @@ -700,18 +697,18 @@ def test_pipe_to_shell_error(base_app, capsys): assert err.startswith("EXCEPTION of type '{}' occurred with message:".format(expected_error)) -@pytest.mark.skipif(not cmd2.can_clip, +@pytest.mark.skipif(not cmd2.cmd2.can_clip, reason="Pyperclip could not find a copy/paste mechanism for your system") def test_send_to_paste_buffer(base_app): # Test writing to the PasteBuffer/Clipboard run_cmd(base_app, 'help >') expected = normalize(BASE_HELP) - assert normalize(cmd2.get_paste_buffer()) == expected + assert normalize(cmd2.cmd2.get_paste_buffer()) == expected # Test appending to the PasteBuffer/Clipboard run_cmd(base_app, 'help history >>') expected = normalize(BASE_HELP + '\n' + HELP_HISTORY) - assert normalize(cmd2.get_paste_buffer()) == expected + assert normalize(cmd2.cmd2.get_paste_buffer()) == expected def test_base_timing(base_app, capsys): @@ -1365,18 +1362,18 @@ optional arguments: """ @pytest.mark.skipif(sys.platform.startswith('win'), - reason="cmd2._which function only used on Mac and Linux") + reason="utils.which function only used on Mac and Linux") def test_which_editor_good(): editor = 'vi' - path = cmd2._which(editor) + path = utils.which(editor) # Assert that the vi editor was found because it should exist on all Mac and Linux systems assert path @pytest.mark.skipif(sys.platform.startswith('win'), - reason="cmd2._which function only used on Mac and Linux") + reason="utils.which function only used on Mac and Linux") def test_which_editor_bad(): editor = 'notepad.exe' - path = cmd2._which(editor) + path = utils.which(editor) # Assert that the editor wasn't found because no notepad.exe on non-Windows systems ;-) assert path is None @@ -1421,7 +1418,7 @@ def test_multiline_complete_statement_without_terminator(multiline_app): def test_clipboard_failure(capsys): # Force cmd2 clipboard to be disabled - cmd2.disable_clip() + cmd2.cmd2.disable_clip() app = cmd2.Cmd() # Redirect command output to the clipboard when a clipboard isn't present @@ -1463,11 +1460,11 @@ def test_cmdresult(cmdresult_app): def test_is_text_file_bad_input(base_app): # Test with a non-existent file - file_is_valid = base_app.is_text_file('does_not_exist.txt') + file_is_valid = utils.is_text_file('does_not_exist.txt') assert not file_is_valid # Test with a directory - dir_is_valid = base_app.is_text_file('.') + dir_is_valid = utils.is_text_file('.') assert not dir_is_valid diff --git a/tests/test_completion.py b/tests/test_completion.py index c7650dbb..2faa4a08 100644 --- a/tests/test_completion.py +++ b/tests/test_completion.py @@ -12,8 +12,8 @@ import argparse import os import sys -from cmd2 import cmd2 import pytest +import cmd2 from .conftest import complete_tester, StdOut from examples.subcommands import SubcommandsExample diff --git a/tests/test_parsing.py b/tests/test_parsing.py index 7b361b7e..59f9a610 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -7,9 +7,8 @@ Released under MIT license, see LICENSE file """ import pytest -from cmd2 import cmd2 +import cmd2 from cmd2.parsing import StatementParser - from cmd2 import utils @pytest.fixture diff --git a/tests/test_transcript.py b/tests/test_transcript.py index 70658161..302d80c8 100644 --- a/tests/test_transcript.py +++ b/tests/test_transcript.py @@ -15,8 +15,9 @@ import tempfile from unittest import mock import pytest -from cmd2 import cmd2 +import cmd2 from .conftest import run_cmd, StdOut +from cmd2 import transcript class CmdLineApp(cmd2.Cmd): @@ -177,7 +178,7 @@ this is a \/multiline\/ command def test_parse_transcript_expected(expected, transformed): app = CmdLineApp() - class TestMyAppCase(cmd2.Cmd2TestCase): + class TestMyAppCase(transcript.Cmd2TestCase): cmdapp = app testcase = TestMyAppCase() |