diff options
| -rw-r--r-- | CHANGELOG | 7 | ||||
| -rw-r--r-- | doc/conf.py | 4 | ||||
| -rw-r--r-- | doc/index.txt | 14 | ||||
| -rw-r--r-- | doc/plugins.txt | 69 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | tests/test_config.py | 219 | ||||
| -rw-r--r-- | tests/test_interpreters.py | 53 | ||||
| -rw-r--r-- | tests/test_venv.py | 7 | ||||
| -rw-r--r-- | tox.ini | 13 | ||||
| -rw-r--r-- | tox/__init__.py | 2 | ||||
| -rw-r--r-- | tox/_cmdline.py | 31 | ||||
| -rw-r--r-- | tox/_config.py | 717 | ||||
| -rw-r--r-- | tox/_venv.py | 26 | ||||
| -rw-r--r-- | tox/hookspecs.py | 32 | ||||
| -rw-r--r-- | tox/interpreters.py | 74 |
15 files changed, 791 insertions, 479 deletions
@@ -38,6 +38,13 @@ - fix issue240: allow to specify empty argument list without it being rewritten to ".". Thanks Daniel Hahler. +- introduce experimental (not much documented yet) plugin system + based on pytest's externalized "pluggy" system. + See tox/hookspecs.py for the current hooks. + +- introduce parser.add_testenv_attribute() to register an ini-variable + for testenv sections. Can be used from plugins through the + tox_add_option hook. 1.9.2 ----------- diff --git a/doc/conf.py b/doc/conf.py index 9c573d7..73f5168 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -48,8 +48,8 @@ copyright = u'2013, holger krekel and others' # built documents. # # The short X.Y version. -release = "1.9" -version = "1.9.0" +release = "2.0" +version = "2.0.0" # The full version, including alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation diff --git a/doc/index.txt b/doc/index.txt index 17d063d..ed594cd 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -5,7 +5,7 @@ vision: standardize testing in Python --------------------------------------------- ``tox`` aims to automate and standardize testing in Python. It is part -of a larger vision of easing the packaging, testing and release process +of a larger vision of easing the packaging, testing and release process of Python software. What is Tox? @@ -21,6 +21,7 @@ Tox is a generic virtualenv_ management and test command line tool you can use f * acting as a frontend to Continuous Integration servers, greatly reducing boilerplate and merging CI and shell-based testing. + Basic example ----------------- @@ -62,10 +63,10 @@ Current features - test-tool agnostic: runs py.test, nose or unittests in a uniform manner -* supports :ref:`using different / multiple PyPI index servers <multiindex>` +* :doc:`(new in 2.0) plugin system <plugins>` to modify tox execution with simple hooks. * uses pip_ and setuptools_ by default. Experimental - support for configuring the installer command + support for configuring the installer command through :confval:`install_command=ARGV`. * **cross-Python compatible**: CPython-2.6, 2.7, 3.2 and higher, @@ -74,11 +75,11 @@ Current features * **cross-platform**: Windows and Unix style environments * **integrates with continuous integration servers** like Jenkins_ - (formerly known as Hudson) and helps you to avoid boilerplatish + (formerly known as Hudson) and helps you to avoid boilerplatish and platform-specific build-step hacks. * **full interoperability with devpi**: is integrated with and - is used for testing in the devpi_ system, a versatile pypi + is used for testing in the devpi_ system, a versatile pypi index server and release managing tool. * **driven by a simple ini-style config file** @@ -89,6 +90,9 @@ Current features * **professionally** :doc:`supported <support>` +* supports :ref:`using different / multiple PyPI index servers <multiindex>` + + .. _pypy: http://pypy.org .. _`tox.ini`: :doc:configfile diff --git a/doc/plugins.txt b/doc/plugins.txt new file mode 100644 index 0000000..61e4408 --- /dev/null +++ b/doc/plugins.txt @@ -0,0 +1,69 @@ +.. be in -*- rst -*- mode! + +tox plugins +=========== + +.. versionadded:: 2.0 + +With tox-2.0 a few aspects of tox running can be experimentally modified +by writing hook functions. We expect the list of hook function to grow +over time. + +writing a setuptools entrypoints plugin +--------------------------------------- + +If you have a ``tox_MYPLUGIN.py`` module you could use the following +rough ``setup.py`` to make it into a package which you can upload to the +Python packaging index:: + + # content of setup.py + from setuptools import setup + + if __name__ == "__main__": + setup( + name='tox-MYPLUGIN', + description='tox plugin decsription', + license="MIT license", + version='0.1', + py_modules=['tox_MYPLUGIN'], + entry_points={'tox': ['MYPLUGIN = tox_MYPLUGIN']}, + install_requires=['tox>=2.0'], + ) + +You can then install the plugin to develop it via:: + + pip install -e . + +and later publish it. + +The ``entry_points`` part allows tox to see your plugin during startup. + + +Writing hook implementations +---------------------------- + +A plugin module needs can define one or more hook implementation functions:: + + from tox import hookimpl + + @hookimpl + def tox_addoption(parser): + # add your own command line options + + + @hookimpl + def tox_configure(config): + # post process tox configuration after cmdline/ini file have + # been parsed + +If you put this into a module and make it pypi-installable with the ``tox`` +entry point you'll get your code executed as part of a tox run. + + + +tox hook specifications +---------------------------- + +.. automodule:: tox.hookspecs + :members: + @@ -18,7 +18,7 @@ class Tox(TestCommand): def main(): version = sys.version_info[:2] - install_requires = ['virtualenv>=1.11.2', 'py>=1.4.17', ] + install_requires = ['virtualenv>=1.11.2', 'py>=1.4.17', 'pluggy>=0.3.0,<0.4.0'] if version < (2, 7): install_requires += ['argparse'] setup( diff --git a/tests/test_config.py b/tests/test_config.py index e4e7bf6..87f7f6d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,6 @@ import pytest import tox import tox._config from tox._config import * # noqa -from tox._config import _split_env from tox._venv import VirtualEnv @@ -19,7 +18,7 @@ class TestVenvConfig: assert config.toxworkdir.realpath() == tmpdir.join(".tox").realpath() assert config.envconfigs['py1'].basepython == sys.executable assert config.envconfigs['py1'].deps == [] - assert not config.envconfigs['py1'].platform + assert config.envconfigs['py1'].platform == ".*" def test_config_parsing_multienv(self, tmpdir, newconfig): config = newconfig([], """ @@ -93,12 +92,12 @@ class TestVenvConfig: """ Ensure correct parseini._is_same_dep is working with a few samples. """ - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') - assert not parseini._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') + assert not DepOption._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') class TestConfigPlatform: @@ -220,8 +219,8 @@ class TestIniParserAgainstCommandsKey: commands = echo {[section]key} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [["echo", "whatever"]] def test_command_substitution_from_other_section_multiline(self, newconfig): @@ -245,8 +244,8 @@ class TestIniParserAgainstCommandsKey: # comment is omitted echo {[base]commands} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [ "cmd1 param11 param12".split(), "cmd2 param21 param22".split(), @@ -257,16 +256,16 @@ class TestIniParserAgainstCommandsKey: class TestIniParser: - def test_getdefault_single(self, tmpdir, newconfig): + def test_getstring_single(self, tmpdir, newconfig): config = newconfig(""" [section] key=value """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key") assert x == "value" - assert not reader.getdefault("section", "hello") - x = reader.getdefault("section", "hello", "world") + assert not reader.getstring("hello") + x = reader.getstring("hello", "world") assert x == "world" def test_missing_substitution(self, tmpdir, newconfig): @@ -274,40 +273,40 @@ class TestIniParser: [mydefault] key2={xyz} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("mydefault", config._cfg, fallbacksections=['mydefault']) assert reader is not None - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("mydefault", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_fallback_sections(self, tmpdir, newconfig): + def test_getstring_fallback_sections(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2=value2 [section] key=value """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) - x = reader.getdefault("section", "key2") + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) + x = reader.getstring("key2") assert x == "value2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "world") + x = reader.getstring("key3", "world") assert x == "world" - def test_getdefault_substitution(self, tmpdir, newconfig): + def test_getstring_substitution(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2={value2} [section] key={value} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) reader.addsubstitutions(value="newvalue", value2="newvalue2") - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "newvalue2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "{value2}") + x = reader.getstring("key3", "{value2}") assert x == "newvalue2" def test_getlist(self, tmpdir, newconfig): @@ -317,9 +316,9 @@ class TestIniParser: item1 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getlist("section", "key2") + x = reader.getlist("key2") assert x == ['item1', 'grr'] def test_getdict(self, tmpdir, newconfig): @@ -329,28 +328,31 @@ class TestIniParser: key1=item1 key2={item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getdict("section", "key2") + x = reader.getdict("key2") assert 'key1' in x assert 'key2' in x assert x['key1'] == 'item1' assert x['key2'] == 'grr' - def test_getdefault_environment_substitution(self, monkeypatch, newconfig): + x = reader.getdict("key3", {1: 2}) + assert x == {1: 2} + + def test_getstring_environment_substitution(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] key1={env:KEY1} key2={env:KEY2} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("section", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_environment_substitution_with_default(self, monkeypatch, newconfig): + def test_getstring_environment_substitution_with_default(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] @@ -358,12 +360,12 @@ class TestIniParser: key2={env:KEY2:DEFAULT_VALUE} key3={env:KEY3:} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "DEFAULT_VALUE" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert x == "" def test_value_matches_section_substituion(self): @@ -374,15 +376,15 @@ class TestIniParser: assert is_section_substitution("{[setup]}") is None assert is_section_substitution("{[setup] commands}") is None - def test_getdefault_other_section_substitution(self, newconfig): + def test_getstring_other_section_substitution(self, newconfig): config = newconfig(""" [section] key = rue [testenv] key = t{[section]key} """) - reader = IniReader(config._cfg) - x = reader.getdefault("testenv", "key") + reader = SectionReader("testenv", config._cfg) + x = reader.getstring("key") assert x == "true" def test_argvlist(self, tmpdir, newconfig): @@ -392,12 +394,12 @@ class TestIniParser: cmd1 {item1} {item2} cmd2 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"], ["cmd2", "grr"]] @@ -406,9 +408,9 @@ class TestIniParser: [section] comm = py.test {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([r"hello\this"]) - argv = reader.getargv("section", "comm") + argv = reader.getargv("comm") assert argv == ["py.test", "hello\\this"] def test_argvlist_multiline(self, tmpdir, newconfig): @@ -418,12 +420,12 @@ class TestIniParser: cmd1 {item1} \ # a comment {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"]] def test_argvlist_quoting_in_command(self, tmpdir, newconfig): @@ -433,8 +435,8 @@ class TestIniParser: cmd1 'with space' \ # a comment 'after the comment' """) - reader = IniReader(config._cfg) - x = reader.getargvlist("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getargvlist("key1") assert x == [["cmd1", "with space", "after the comment"]] def test_argvlist_positional_substitution(self, tmpdir, newconfig): @@ -445,22 +447,22 @@ class TestIniParser: cmd2 {posargs:{item2} \ other} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] + posargs assert argvlist[1] == ["cmd2"] + posargs - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([], item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] assert argvlist[1] == ["cmd2", "value2", "other"] @@ -472,10 +474,10 @@ class TestIniParser: cmd2 -f '{posargs}' cmd3 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "bar"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "--foo-args=foo bar"], ["cmd2", "-f", "foo bar"], ["cmd3", "-f", "foo", "bar"]] @@ -486,10 +488,10 @@ class TestIniParser: key2= cmd1 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "'bar", "baz'"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "-f", "foo", "bar baz"]] def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig): @@ -501,11 +503,11 @@ class TestIniParser: cmd2 -m '\'something\'' [] cmd3 something[]else """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs) - argvlist = reader.getargvlist('section', 'key') + argvlist = reader.getargvlist('key') assert argvlist[0] == ['cmd0'] + posargs assert argvlist[1] == ['cmd1', '-m', '[abc]'] assert argvlist[2] == ['cmd2', '-m', "something"] + posargs @@ -517,32 +519,32 @@ class TestIniParser: key = py.test -n5 --junitxml={envlogdir}/junit-{envname}.xml [] """ config = newconfig(inisource) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, envlogdir='ENV_LOG_DIR', envname='ENV_NAME') expected = [ 'py.test', '-n5', '--junitxml=ENV_LOG_DIR/junit-ENV_NAME.xml', 'hello', 'world' ] - assert reader.getargvlist('section', 'key')[0] == expected + assert reader.getargvlist('key')[0] == expected def test_getargv(self, newconfig): config = newconfig(""" [section] key=some command "with quoting" """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) expected = ['some', 'command', 'with quoting'] - assert reader.getargv('section', 'key') == expected + assert reader.getargv('key') == expected def test_getpath(self, tmpdir, newconfig): config = newconfig(""" [section] path1={HELLO} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(toxinidir=tmpdir, HELLO="mypath") - x = reader.getpath("section", "path1", tmpdir) + x = reader.getpath("path1", tmpdir) assert x == tmpdir.join("mypath") def test_getbool(self, tmpdir, newconfig): @@ -554,13 +556,13 @@ class TestIniParser: key2a=falsE key5=yes """) - reader = IniReader(config._cfg) - assert reader.getbool("section", "key1") is True - assert reader.getbool("section", "key1a") is True - assert reader.getbool("section", "key2") is False - assert reader.getbool("section", "key2a") is False - py.test.raises(KeyError, 'reader.getbool("section", "key3")') - py.test.raises(tox.exception.ConfigError, 'reader.getbool("section", "key5")') + reader = SectionReader("section", config._cfg) + assert reader.getbool("key1") is True + assert reader.getbool("key1a") is True + assert reader.getbool("key2") is False + assert reader.getbool("key2a") is False + py.test.raises(KeyError, 'reader.getbool("key3")') + py.test.raises(tox.exception.ConfigError, 'reader.getbool("key5")') class TestConfigTestEnv: @@ -586,7 +588,7 @@ class TestConfigTestEnv: assert envconfig.commands == [["xyz", "--abc"]] assert envconfig.changedir == config.setupdir assert envconfig.sitepackages is False - assert envconfig.develop is False + assert envconfig.usedevelop is False assert envconfig.ignore_errors is False assert envconfig.envlogdir == envconfig.envdir.join("log") assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] @@ -607,7 +609,7 @@ class TestConfigTestEnv: [testenv] usedevelop = True """) - assert not config.envconfigs["python"].develop + assert not config.envconfigs["python"].usedevelop def test_specific_command_overrides(self, tmpdir, newconfig): config = newconfig(""" @@ -1382,7 +1384,7 @@ class TestHashseedOption: def test_noset(self, tmpdir, newconfig): args = ['--hashseed', 'noset'] envconfig = self._get_envconfig(newconfig, args=args) - assert envconfig.setenv is None + assert envconfig.setenv == {} def test_noset_with_setenv(self, tmpdir, newconfig): tox_ini = """ @@ -1571,31 +1573,16 @@ class TestCmdInvocation: ]) -class TestArgumentParser: - - def test_dash_e_single_1(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26'] - - def test_dash_e_single_2(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26,py33'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26', 'py33'] - - def test_dash_e_same(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26,py26'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26', 'py26'] - - def test_dash_e_combine(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26,py25,py33 -e py33,py27'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26', 'py25', 'py33', 'py33', 'py27'] +@pytest.mark.parametrize("cmdline,envlist", [ + ("-e py26", ['py26']), + ("-e py26,py33", ['py26', 'py33']), + ("-e py26,py26", ['py26', 'py26']), + ("-e py26,py33 -e py33,py27", ['py26', 'py33', 'py33', 'py27']) +]) +def test_env_spec(cmdline, envlist): + args = cmdline.split() + config = parseconfig(args) + assert config.envlist == envlist class TestCommandParser: diff --git a/tests/test_interpreters.py b/tests/test_interpreters.py index 1c5a77d..01d6508 100644 --- a/tests/test_interpreters.py +++ b/tests/test_interpreters.py @@ -3,11 +3,13 @@ import os import pytest from tox.interpreters import * # noqa +from tox._config import get_plugin_manager @pytest.fixture def interpreters(): - return Interpreters() + pm = get_plugin_manager() + return Interpreters(hook=pm.hook) @pytest.mark.skipif("sys.platform != 'win32'") @@ -28,8 +30,11 @@ def test_locate_via_py(monkeypatch): assert locate_via_py('3', '2') == sys.executable -def test_find_executable(): - p = find_executable(sys.executable) +def test_tox_get_python_executable(): + class envconfig: + basepython = sys.executable + envname = "pyxx" + p = tox_get_python_executable(envconfig) assert p == py.path.local(sys.executable) for ver in [""] + "2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3".split(): name = "python%s" % ver @@ -42,7 +47,8 @@ def test_find_executable(): else: if not py.path.local.sysfind(name): continue - p = find_executable(name) + envconfig.basepython = name + p = tox_get_python_executable(envconfig) assert p popen = py.std.subprocess.Popen([str(p), '-V'], stderr=py.std.subprocess.PIPE) @@ -55,7 +61,12 @@ def test_find_executable_extra(monkeypatch): def sysfind(x): return "hello" monkeypatch.setattr(py.path.local, "sysfind", sysfind) - t = find_executable("qweqwe") + + class envconfig: + basepython = "1lk23j" + envname = "pyxx" + + t = tox_get_python_executable(envconfig) assert t == "hello" @@ -69,31 +80,33 @@ def test_run_and_get_interpreter_info(): class TestInterpreters: - def test_get_info_self_exceptions(self, interpreters): - pytest.raises(ValueError, lambda: - interpreters.get_info()) - pytest.raises(ValueError, lambda: - interpreters.get_info(name="12", executable="123")) - def test_get_executable(self, interpreters): - x = interpreters.get_executable(sys.executable) - assert x == sys.executable - assert not interpreters.get_executable("12l3k1j23") + class envconfig: + basepython = sys.executable + envname = "pyxx" - def test_get_info__name(self, interpreters): - info = interpreters.get_info(executable=sys.executable) + x = interpreters.get_executable(envconfig) + assert x == sys.executable + info = interpreters.get_info(envconfig) assert info.version_info == tuple(sys.version_info) assert info.executable == sys.executable assert info.runnable - def test_get_info__name_not_exists(self, interpreters): - info = interpreters.get_info("qlwkejqwe") + def test_get_executable_no_exist(self, interpreters): + class envconfig: + basepython = "1lkj23" + envname = "pyxx" + assert not interpreters.get_executable(envconfig) + info = interpreters.get_info(envconfig) assert not info.version_info - assert info.name == "qlwkejqwe" + assert info.name == "1lkj23" assert not info.executable assert not info.runnable def test_get_sitepackagesdir_error(self, interpreters): - info = interpreters.get_info(sys.executable) + class envconfig: + basepython = sys.executable + envname = "123" + info = interpreters.get_info(envconfig) s = interpreters.get_sitepackagesdir(info, "") assert s diff --git a/tests/test_venv.py b/tests/test_venv.py index 538b7e5..b32affd 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -35,6 +35,7 @@ def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): py.test.raises(tox.exception.UnsupportedInterpreter, venv.getsupportedinterpreter) monkeypatch.undo() + monkeypatch.setattr(venv.envconfig, "envname", "py1") monkeypatch.setattr(venv.envconfig, 'basepython', 'notexistingpython') py.test.raises(tox.exception.InterpreterNotFound, venv.getsupportedinterpreter) @@ -42,7 +43,7 @@ def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): # check that we properly report when no version_info is present info = NoInterpreterInfo(name=venv.name) info.executable = "something" - monkeypatch.setattr(config.interpreters, "get_info", lambda *args: info) + monkeypatch.setattr(config.interpreters, "get_info", lambda *args, **kw: info) pytest.raises(tox.exception.InvocationError, venv.getsupportedinterpreter) @@ -65,7 +66,7 @@ def test_create(monkeypatch, mocksession, newconfig): # assert Envconfig.toxworkdir in args assert venv.getcommandpath("easy_install", cwd=py.path.local()) interp = venv._getliveconfig().python - assert interp == venv.envconfig._basepython_info.executable + assert interp == venv.envconfig.python_info.executable assert venv.path_config.check(exists=False) @@ -463,7 +464,7 @@ class TestCreationConfig: venv = VirtualEnv(envconfig, session=mocksession) venv.update() cconfig = venv._getliveconfig() - cconfig.develop = True + cconfig.usedevelop = True cconfig.writeconfig(venv.path_config) mocksession._clearmocks() venv.update() @@ -5,8 +5,10 @@ envlist=py27,py26,py34,py33,pypy,flakes commands=echo {posargs} [testenv] -commands=py.test --junitxml={envlogdir}/junit-{envname}.xml {posargs} +commands= py.test --timeout=60 {posargs} + deps=pytest>=2.3.5 + pytest-timeout [testenv:docs] basepython=python @@ -14,15 +16,16 @@ changedir=doc deps=sphinx {[testenv]deps} commands= - py.test -v \ - --junitxml={envlogdir}/junit-{envname}.xml \ - check_sphinx.py {posargs} + py.test -v check_sphinx.py {posargs} [testenv:flakes] +qwe = 123 deps = pytest-flakes>=0.2 pytest-pep8 -commands = py.test -x --flakes --pep8 tox tests +commands = + py.test --flakes -m flakes tox tests + py.test --pep8 -m pep8 tox tests [testenv:dev] # required to make looponfail reload on every source code change diff --git a/tox/__init__.py b/tox/__init__.py index 7869fcb..5f3f3ad 100644 --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,6 +1,8 @@ # __version__ = '2.0.0.dev1' +from .hookspecs import hookspec, hookimpl # noqa + class exception: class Error(Exception): diff --git a/tox/_cmdline.py b/tox/_cmdline.py index 43fe5bc..2a9f391 100644 --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -24,13 +24,35 @@ def now(): def main(args=None): try: - config = parseconfig(args, 'tox') + config = parseconfig(args) + if config.option.help: + show_help(config) + raise SystemExit(0) + elif config.option.helpini: + show_help_ini(config) + raise SystemExit(0) retcode = Session(config).runcommand() raise SystemExit(retcode) except KeyboardInterrupt: raise SystemExit(2) +def show_help(config): + tw = py.io.TerminalWriter() + tw.write(config._parser.format_help()) + tw.line() + + +def show_help_ini(config): + tw = py.io.TerminalWriter() + tw.sep("-", "per-testenv attributes") + for env_attr in config._testenv_attr: + tw.line("%-15s %-8s default: %s" % + (env_attr.name, "<" + env_attr.type + ">", env_attr.default), bold=True) + tw.line(env_attr.help) + tw.line() + + class Action(object): def __init__(self, session, venv, msg, args): self.venv = venv @@ -487,7 +509,7 @@ class Session: venv.status = "platform mismatch" continue # we simply omit non-matching platforms if self.setupenv(venv): - if venv.envconfig.develop: + if venv.envconfig.usedevelop: self.developpkg(venv, self.config.setupdir) elif self.config.skipsdist or venv.envconfig.skip_install: self.finishvenv(venv) @@ -551,8 +573,7 @@ class Session: for envconfig in self.config.envconfigs.values(): self.report.line("[testenv:%s]" % envconfig.envname, bold=True) self.report.line(" basepython=%s" % envconfig.basepython) - self.report.line(" _basepython_info=%s" % - envconfig._basepython_info) + self.report.line(" pythoninfo=%s" % (envconfig.python_info,)) self.report.line(" envpython=%s" % envconfig.envpython) self.report.line(" envtmpdir=%s" % envconfig.envtmpdir) self.report.line(" envbindir=%s" % envconfig.envbindir) @@ -567,7 +588,7 @@ class Session: self.report.line(" deps=%s" % envconfig.deps) self.report.line(" envdir= %s" % envconfig.envdir) self.report.line(" downloadcache=%s" % envconfig.downloadcache) - self.report.line(" usedevelop=%s" % envconfig.develop) + self.report.line(" usedevelop=%s" % envconfig.usedevelop) def showenvs(self): for env in self.config.envlist: diff --git a/tox/_config.py b/tox/_config.py index afb6544..cbbf5e9 100644 --- a/tox/_config.py +++ b/tox/_config.py @@ -8,8 +8,10 @@ import shlex import string import pkg_resources import itertools +import pluggy -from tox.interpreters import Interpreters +import tox.interpreters +from tox import hookspecs import py @@ -22,20 +24,161 @@ default_factors = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3', for version in '24,25,26,27,30,31,32,33,34,35'.split(','): default_factors['py' + version] = 'python%s.%s' % tuple(version) +hookimpl = pluggy.HookimplMarker("tox") -def parseconfig(args=None, pkg=None): + +def get_plugin_manager(): + # initialize plugin manager + pm = pluggy.PluginManager("tox") + pm.add_hookspecs(hookspecs) + pm.register(tox._config) + pm.register(tox.interpreters) + pm.load_setuptools_entrypoints("tox") + pm.check_pending() + return pm + + +class MyParser: + def __init__(self): + self.argparser = argparse.ArgumentParser( + description="tox options", add_help=False) + self._testenv_attr = [] + + def add_argument(self, *args, **kwargs): + return self.argparser.add_argument(*args, **kwargs) + + def add_testenv_attribute(self, name, type, help, default=None, postprocess=None): + self._testenv_attr.append(VenvAttribute(name, type, default, help, postprocess)) + + def add_testenv_attribute_obj(self, obj): + assert hasattr(obj, "name") + assert hasattr(obj, "type") + assert hasattr(obj, "help") + assert hasattr(obj, "postprocess") + self._testenv_attr.append(obj) + + def parse_args(self, args): + return self.argparser.parse_args(args) + + def format_help(self): + return self.argparser.format_help() + + +class VenvAttribute: + def __init__(self, name, type, default, help, postprocess): + self.name = name + self.type = type + self.default = default + self.help = help + self.postprocess = postprocess + + +class DepOption: + name = "deps" + type = "line-list" + help = "each line specifies a dependency in pip/setuptools format." + default = () + + def postprocess(self, config, reader, section_val): + deps = [] + for depline in section_val: + m = re.match(r":(\w+):\s*(\S+)", depline) + if m: + iname, name = m.groups() + ixserver = config.indexserver[iname] + else: + name = depline.strip() + ixserver = None + name = self._replace_forced_dep(name, config) + deps.append(DepConfig(name, ixserver)) + return deps + + def _replace_forced_dep(self, name, config): + """ + Override the given dependency config name taking --force-dep-version + option into account. + + :param name: dep config, for example ["pkg==1.0", "other==2.0"]. + :param config: Config instance + :return: the new dependency that should be used for virtual environments + """ + if not config.option.force_dep: + return name + for forced_dep in config.option.force_dep: + if self._is_same_dep(forced_dep, name): + return forced_dep + return name + + @classmethod + def _is_same_dep(cls, dep1, dep2): + """ + Returns True if both dependency definitions refer to the + same package, even if versions differ. + """ + dep1_name = pkg_resources.Requirement.parse(dep1).project_name + dep2_name = pkg_resources.Requirement.parse(dep2).project_name + return dep1_name == dep2_name + + +class PosargsOption: + name = "args_are_paths" + type = "bool" + default = True + help = "treat positional args in commands as paths" + + def postprocess(self, config, reader, section_val): + args = config.option.args + if args: + if section_val: + args = [] + for arg in config.option.args: + if arg: + origpath = config.invocationcwd.join(arg, abs=True) + if origpath.check(): + arg = reader.getpath("changedir", ".").bestrelpath(origpath) + args.append(arg) + reader.addsubstitutions(args) + return section_val + + +class InstallcmdOption: + name = "install_command" + type = "argv" + default = "pip install {opts} {packages}" + help = "install command for dependencies and package under test." + + def postprocess(self, config, reader, section_val): + if '{packages}' not in section_val: + raise tox.exception.ConfigError( + "'install_command' must contain '{packages}' substitution") + return section_val + + +def parseconfig(args=None): """ :param list[str] args: Optional list of arguments. :type pkg: str :rtype: :class:`Config` :raise SystemExit: toxinit file is not found """ + + pm = get_plugin_manager() + if args is None: args = sys.argv[1:] - parser = prepare_parse(pkg) - opts = parser.parse_args(args) - config = Config() - config.option = opts + + # prepare command line options + parser = MyParser() + pm.hook.tox_addoption(parser=parser) + + # parse command line options + option = parser.parse_args(args) + interpreters = tox.interpreters.Interpreters(hook=pm.hook) + config = Config(pluginmanager=pm, option=option, interpreters=interpreters) + config._parser = parser + config._testenv_attr = parser._testenv_attr + + # parse ini file basename = config.option.configfile if os.path.isabs(basename): inipath = py.path.local(basename) @@ -52,6 +195,10 @@ def parseconfig(args=None, pkg=None): exn = sys.exc_info()[1] # Use stdout to match test expectations py.builtin.print_("ERROR: " + str(exn)) + + # post process config object + pm.hook.tox_configure(config=config) + return config @@ -63,10 +210,8 @@ def feedback(msg, sysexit=False): class VersionAction(argparse.Action): def __call__(self, argparser, *args, **kwargs): - name = argparser.pkgname - mod = __import__(name) - version = mod.__version__ - py.builtin.print_("%s imported from %s" % (version, mod.__file__)) + version = tox.__version__ + py.builtin.print_("%s imported from %s" % (version, tox.__file__)) raise SystemExit(0) @@ -78,13 +223,16 @@ class CountAction(argparse.Action): setattr(namespace, self.dest, 0) -def prepare_parse(pkgname): - parser = argparse.ArgumentParser(description=__doc__,) +@hookimpl +def tox_addoption(parser): # formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.pkgname = pkgname parser.add_argument("--version", nargs=0, action=VersionAction, dest="version", help="report version information to stdout.") + parser.add_argument("-h", "--help", action="store_true", dest="help", + help="show help about options") + parser.add_argument("--help-ini", "--hi", action="store_true", dest="helpini", + help="show help about ini-names") parser.add_argument("-v", nargs=0, action=CountAction, default=0, dest="verbosity", help="increase verbosity of reporting output.") @@ -130,6 +278,7 @@ def prepare_parse(pkgname): "all commands and results involved. This will turn off " "pass-through output from running test commands which is " "instead captured into the json result file.") + # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. parser.add_argument("--hashseed", action="store", metavar="SEED", default=None, @@ -149,14 +298,144 @@ def prepare_parse(pkgname): parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") - return parser + + # add various core venv interpreter attributes + + parser.add_testenv_attribute( + name="envdir", type="path", default="{toxworkdir}/{envname}", + help="venv directory") + + parser.add_testenv_attribute( + name="envtmpdir", type="path", default="{envdir}/tmp", + help="venv temporary directory") + + parser.add_testenv_attribute( + name="envlogdir", type="path", default="{envdir}/log", + help="venv log directory") + + def downloadcache(config, reader, section_val): + if section_val: + # env var, if present, takes precedence + downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", section_val) + return py.path.local(downloadcache) + + parser.add_testenv_attribute( + name="downloadcache", type="string", default=None, postprocess=downloadcache, + help="(deprecated) set PIP_DOWNLOAD_CACHE.") + + parser.add_testenv_attribute( + name="changedir", type="path", default="{toxinidir}", + help="directory to change to when running commands") + + parser.add_testenv_attribute_obj(PosargsOption()) + + parser.add_testenv_attribute( + name="skip_install", type="bool", default=False, + help="Do not install the current package. This can be used when " + "you need the virtualenv management but do not want to install " + "the current package") + + parser.add_testenv_attribute( + name="ignore_errors", type="bool", default=False, + help="if set to True all commands will be executed irrespective of their " + "result error status.") + + def recreate(config, reader, section_val): + if config.option.recreate: + return True + return section_val + + parser.add_testenv_attribute( + name="recreate", type="bool", default=False, postprocess=recreate, + help="always recreate this test environment.") + + def setenv(config, reader, section_val): + setenv = section_val + if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: + setenv['PYTHONHASHSEED'] = config.hashseed + return setenv + + parser.add_testenv_attribute( + name="setenv", type="dict", postprocess=setenv, + help="list of X=Y lines with environment variable settings") + + def passenv(config, reader, section_val): + passenv = set(["PATH"]) + if sys.platform == "win32": + passenv.add("SYSTEMROOT") # needed for python's crypto module + passenv.add("PATHEXT") # needed for discovering executables + for spec in section_val: + for name in os.environ: + if fnmatchcase(name.upper(), spec.upper()): + passenv.add(name) + return passenv + + parser.add_testenv_attribute( + name="passenv", type="space-separated-list", postprocess=passenv, + help="environment variables names which shall be passed " + "from tox invocation to test environment when executing commands.") + + parser.add_testenv_attribute( + name="whitelist_externals", type="line-list", + help="each lines specifies a path or basename for which tox will not warn " + "about it coming from outside the test environment.") + + parser.add_testenv_attribute( + name="platform", type="string", default=".*", + help="regular expression which must match against ``sys.platform``. " + "otherwise testenv will be skipped.") + + def sitepackages(config, reader, section_val): + return config.option.sitepackages or section_val + + parser.add_testenv_attribute( + name="sitepackages", type="bool", default=False, postprocess=sitepackages, + help="Set to ``True`` if you want to create virtual environments that also " + "have access to globally installed packages.") + + def pip_pre(config, reader, section_val): + return config.option.pre or section_val + + parser.add_testenv_attribute( + name="pip_pre", type="bool", default=False, postprocess=pip_pre, + help="If ``True``, adds ``--pre`` to the ``opts`` passed to " + "the install command. ") + + def develop(config, reader, section_val): + return not config.option.installpkg and (section_val or config.option.develop) + + parser.add_testenv_attribute( + name="usedevelop", type="bool", postprocess=develop, default=False, + help="install package in develop/editable mode") + + def basepython_default(config, reader, section_val): + if section_val is None: + for f in reader.factors: + if f in default_factors: + return default_factors[f] + return sys.executable + return str(section_val) + + parser.add_testenv_attribute( + name="basepython", type="string", default=None, postprocess=basepython_default, + help="executable name or path of interpreter used to create a " + "virtual test environment.") + + parser.add_testenv_attribute_obj(InstallcmdOption()) + parser.add_testenv_attribute_obj(DepOption()) + + parser.add_testenv_attribute( + name="commands", type="argvlist", default="", + help="each line specifies a test command and can use substitution.") class Config(object): - def __init__(self): + def __init__(self, pluginmanager, option, interpreters): self.envconfigs = {} self.invocationcwd = py.path.local() - self.interpreters = Interpreters() + self.interpreters = interpreters + self.pluginmanager = pluginmanager + self.option = option @property def homedir(self): @@ -192,16 +471,20 @@ class VenvConfig: def envsitepackagesdir(self): self.getsupportedinterpreter() # for throwing exceptions x = self.config.interpreters.get_sitepackagesdir( - info=self._basepython_info, + info=self.python_info, envdir=self.envdir) return x + @property + def python_info(self): + return self.config.interpreters.get_info(envconfig=self) + def getsupportedinterpreter(self): if sys.platform == "win32" and self.basepython and \ "jython" in self.basepython: raise tox.exception.UnsupportedInterpreter( "Jython/Windows does not support installing scripts") - info = self.config.interpreters.get_info(self.basepython) + info = self.config.interpreters.get_info(envconfig=self) if not info.executable: raise tox.exception.InterpreterNotFound(self.basepython) if not info.version_info: @@ -240,12 +523,10 @@ class parseini: self.config = config ctxname = getcontextname() if ctxname == "jenkins": - reader = IniReader(self._cfg, fallbacksections=['tox']) - toxsection = "tox:%s" % ctxname + reader = SectionReader("tox:jenkins", self._cfg, fallbacksections=['tox']) distshare_default = "{toxworkdir}/distshare" elif not ctxname: - reader = IniReader(self._cfg) - toxsection = "tox" + reader = SectionReader("tox", self._cfg) distshare_default = "{homedir}/.tox/distshare" else: raise ValueError("invalid context") @@ -260,18 +541,17 @@ class parseini: reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) - config.toxworkdir = reader.getpath(toxsection, "toxworkdir", - "{toxinidir}/.tox") - config.minversion = reader.getdefault(toxsection, "minversion", None) + config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox") + config.minversion = reader.getstring("minversion", None) if not config.option.skip_missing_interpreters: config.option.skip_missing_interpreters = \ - reader.getbool(toxsection, "skip_missing_interpreters", False) + reader.getbool("skip_missing_interpreters", False) # determine indexserver dictionary config.indexserver = {'default': IndexServerConfig('default')} prefix = "indexserver" - for line in reader.getlist(toxsection, prefix): + for line in reader.getlist(prefix): name, url = map(lambda x: x.strip(), line.split("=", 1)) config.indexserver[name] = IndexServerConfig(name, url) @@ -296,16 +576,15 @@ class parseini: config.indexserver[name] = IndexServerConfig(name, override) reader.addsubstitutions(toxworkdir=config.toxworkdir) - config.distdir = reader.getpath(toxsection, "distdir", "{toxworkdir}/dist") + config.distdir = reader.getpath("distdir", "{toxworkdir}/dist") reader.addsubstitutions(distdir=config.distdir) - config.distshare = reader.getpath(toxsection, "distshare", - distshare_default) + config.distshare = reader.getpath("distshare", distshare_default) reader.addsubstitutions(distshare=config.distshare) - config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None) - config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}") + config.sdistsrc = reader.getpath("sdistsrc", None) + config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") - config.envlist, all_envs = self._getenvdata(reader, toxsection) + config.envlist, all_envs = self._getenvdata(reader) # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -313,7 +592,7 @@ class parseini: known_factors.add("python") # factors stated in config envlist - stated_envlist = reader.getdefault(toxsection, "envlist", replace=False) + stated_envlist = reader.getstring("envlist", replace=False) if stated_envlist: for env in _split_env(stated_envlist): known_factors.update(env.split('-')) @@ -324,13 +603,13 @@ class parseini: factors = set(name.split('-')) if section in self._cfg or factors <= known_factors: config.envconfigs[name] = \ - self._makeenvconfig(name, section, reader._subs, config) + self.make_envconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs - and config.envconfigs[name].develop + and config.envconfigs[name].usedevelop for name in config.envlist) - config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop) + config.skipsdist = reader.getbool("skipsdist", all_develop) def _list_section_factors(self, section): factors = set() @@ -340,116 +619,42 @@ class parseini: factors.update(*mapcat(_split_factor_expr, exprs)) return factors - def _makeenvconfig(self, name, section, subs, config): + def make_envconfig(self, name, section, subs, config): vc = VenvConfig(config=config, envname=name) factors = set(name.split('-')) - reader = IniReader(self._cfg, fallbacksections=["testenv"], factors=factors) + reader = SectionReader(section, self._cfg, fallbacksections=["testenv"], + factors=factors) reader.addsubstitutions(**subs) - vc.develop = ( - not config.option.installpkg - and reader.getbool(section, "usedevelop", config.option.develop)) - vc.envdir = reader.getpath(section, "envdir", "{toxworkdir}/%s" % name) - vc.args_are_paths = reader.getbool(section, "args_are_paths", True) - if reader.getdefault(section, "python", None): - raise tox.exception.ConfigError( - "'python=' key was renamed to 'basepython='") - bp = next((default_factors[f] for f in factors if f in default_factors), - sys.executable) - vc.basepython = reader.getdefault(section, "basepython", bp) - vc._basepython_info = config.interpreters.get_info(vc.basepython) - reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname, - envbindir=vc.envbindir, envpython=vc.envpython, - envsitepackagesdir=vc.envsitepackagesdir) - vc.envtmpdir = reader.getpath(section, "tmpdir", "{envdir}/tmp") - vc.envlogdir = reader.getpath(section, "envlogdir", "{envdir}/log") - reader.addsubstitutions(envlogdir=vc.envlogdir, envtmpdir=vc.envtmpdir) - vc.changedir = reader.getpath(section, "changedir", "{toxinidir}") - if config.option.recreate: - vc.recreate = True - else: - vc.recreate = reader.getbool(section, "recreate", False) - args = config.option.args - if args: - if vc.args_are_paths: - args = [] - for arg in config.option.args: - if arg: - origpath = config.invocationcwd.join(arg, abs=True) - if origpath.check(): - arg = vc.changedir.bestrelpath(origpath) - args.append(arg) - reader.addsubstitutions(args) - setenv = {} - if config.hashseed is not None: - setenv['PYTHONHASHSEED'] = config.hashseed - setenv.update(reader.getdict(section, 'setenv')) - - # read passenv - vc.passenv = set(["PATH"]) - if sys.platform == "win32": - vc.passenv.add("SYSTEMROOT") # needed for python's crypto module - vc.passenv.add("PATHEXT") # needed for discovering executables - for spec in reader.getlist(section, "passenv", sep=" "): - for name in os.environ: - if fnmatchcase(name.lower(), spec.lower()): - vc.passenv.add(name) - - vc.setenv = setenv - if not vc.setenv: - vc.setenv = None - - vc.commands = reader.getargvlist(section, "commands") - vc.whitelist_externals = reader.getlist(section, - "whitelist_externals") - vc.deps = [] - for depline in reader.getlist(section, "deps"): - m = re.match(r":(\w+):\s*(\S+)", depline) - if m: - iname, name = m.groups() - ixserver = config.indexserver[iname] + reader.addsubstitutions(envname=name) + + for env_attr in config._testenv_attr: + atype = env_attr.type + if atype in ("bool", "path", "string", "dict", "argv", "argvlist"): + meth = getattr(reader, "get" + atype) + res = meth(env_attr.name, env_attr.default) + elif atype == "space-separated-list": + res = reader.getlist(env_attr.name, sep=" ") + elif atype == "line-list": + res = reader.getlist(env_attr.name, sep="\n") else: - name = depline.strip() - ixserver = None - name = self._replace_forced_dep(name, config) - vc.deps.append(DepConfig(name, ixserver)) + raise ValueError("unknown type %r" % (atype,)) - platform = "" - for platform in reader.getlist(section, "platform"): - if platform.strip(): - break - vc.platform = platform - - vc.sitepackages = ( - self.config.option.sitepackages - or reader.getbool(section, "sitepackages", False)) - - vc.downloadcache = None - downloadcache = reader.getdefault(section, "downloadcache") - if downloadcache: - # env var, if present, takes precedence - downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", downloadcache) - vc.downloadcache = py.path.local(downloadcache) - - vc.install_command = reader.getargv( - section, - "install_command", - "pip install {opts} {packages}", - ) - if '{packages}' not in vc.install_command: - raise tox.exception.ConfigError( - "'install_command' must contain '{packages}' substitution") - vc.pip_pre = config.option.pre or reader.getbool( - section, "pip_pre", False) + if env_attr.postprocess: + res = env_attr.postprocess(config, reader, res) + setattr(vc, env_attr.name, res) - vc.skip_install = reader.getbool(section, "skip_install", False) - vc.ignore_errors = reader.getbool(section, "ignore_errors", False) + if atype == "path": + reader.addsubstitutions(**{env_attr.name: res}) + if env_attr.name == "install_command": + reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, + envsitepackagesdir=vc.envsitepackagesdir) return vc - def _getenvdata(self, reader, toxsection): + def _getenvdata(self, reader): envstr = self.config.option.env \ or os.environ.get("TOXENV") \ - or reader.getdefault(toxsection, "envlist", replace=False) \ + or reader.getstring("envlist", replace=False) \ or [] envlist = _split_env(envstr) @@ -466,32 +671,6 @@ class parseini: return envlist, all_envs - def _replace_forced_dep(self, name, config): - """ - Override the given dependency config name taking --force-dep-version - option into account. - - :param name: dep config, for example ["pkg==1.0", "other==2.0"]. - :param config: Config instance - :return: the new dependency that should be used for virtual environments - """ - if not config.option.force_dep: - return name - for forced_dep in config.option.force_dep: - if self._is_same_dep(forced_dep, name): - return forced_dep - return name - - @classmethod - def _is_same_dep(cls, dep1, dep2): - """ - Returns True if both dependency definitions refer to the - same package, even if versions differ. - """ - dep1_name = pkg_resources.Requirement.parse(dep1).project_name - dep2_name = pkg_resources.Requirement.parse(dep2).project_name - return dep1_name == dep2_name - def _split_env(env): """if handed a list, action="append" was used for -e """ @@ -558,8 +737,9 @@ RE_ITEM_REF = re.compile( re.VERBOSE) -class IniReader: - def __init__(self, cfgparser, fallbacksections=None, factors=()): +class SectionReader: + def __init__(self, section_name, cfgparser, fallbacksections=None, factors=()): + self.section_name = section_name self._cfg = cfgparser self.fallbacksections = fallbacksections or [] self.factors = factors @@ -571,129 +751,39 @@ class IniReader: if _posargs: self.posargs = _posargs - def getpath(self, section, name, defaultpath): + def getpath(self, name, defaultpath): toxinidir = self._subs['toxinidir'] - path = self.getdefault(section, name, defaultpath) + path = self.getstring(name, defaultpath) if path is None: return path return toxinidir.join(path, abs=True) - def getlist(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getlist(self, name, sep="\n"): + s = self.getstring(name, None) if s is None: return [] return [x.strip() for x in s.split(sep) if x.strip()] - def getdict(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getdict(self, name, default=None, sep="\n"): + s = self.getstring(name, None) if s is None: - return {} + return default or {} value = {} for line in s.split(sep): - if not line.strip(): - continue - name, rest = line.split('=', 1) - value[name.strip()] = rest.strip() + if line.strip(): + name, rest = line.split('=', 1) + value[name.strip()] = rest.strip() return value - def getargvlist(self, section, name): - """Get arguments for every parsed command. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - content = self.getdefault(section, name, '', replace=False) - return self._parse_commands(section, name, content) - - def _parse_commands(self, section, name, content): - """Parse commands from key content in specified section. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :param str content: Content stored by key. - - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - commands = [] - current_command = "" - for line in content.splitlines(): - line = line.rstrip() - i = line.find("#") - if i != -1: - line = line[:i].rstrip() - if not line: - continue - if line.endswith("\\"): - current_command += " " + line[:-1] - continue - current_command += line - - if is_section_substitution(current_command): - replaced = self._replace(current_command) - commands.extend(self._parse_commands(section, name, replaced)) - else: - commands.append(self._processcommand(current_command)) - current_command = "" - else: - if current_command: - raise tox.exception.ConfigError( - "line-continuation ends nowhere while resolving for [%s] %s" % - (section, name)) - return commands - - def _processcommand(self, command): - posargs = getattr(self, "posargs", None) - - # Iterate through each word of the command substituting as - # appropriate to construct the new command string. This - # string is then broken up into exec argv components using - # shlex. - newcommand = "" - for word in CommandParser(command).words(): - if word == "{posargs}" or word == "[]": - if posargs: - newcommand += " ".join(posargs) - continue - elif word.startswith("{posargs:") and word.endswith("}"): - if posargs: - newcommand += " ".join(posargs) - continue - else: - word = word[9:-1] - new_arg = "" - new_word = self._replace(word) - new_word = self._replace(new_word) - new_arg += new_word - newcommand += new_arg - - # Construct shlex object that will not escape any values, - # use all values as is in argv. - shlexer = shlex.shlex(newcommand, posix=True) - shlexer.whitespace_split = True - shlexer.escape = '' - shlexer.commenters = '' - argv = list(shlexer) - return argv - - def getargv(self, section, name, default=None, replace=True): - command = self.getdefault( - section, name, default=default, replace=False) - return self._processcommand(command.strip()) - - def getbool(self, section, name, default=None): - s = self.getdefault(section, name, default) + def getbool(self, name, default=None): + s = self.getstring(name, default) if not s: s = default if s is None: raise KeyError("no config value [%s] %s found" % ( - section, name)) + self.section_name, name)) if not isinstance(s, bool): if s.lower() == "true": @@ -705,9 +795,16 @@ class IniReader: "boolean value %r needs to be 'True' or 'False'") return s - def getdefault(self, section, name, default=None, replace=True): + def getargvlist(self, name, default=""): + s = self.getstring(name, default, replace=False) + return _ArgvlistReader.getargvlist(self, s) + + def getargv(self, name, default=""): + return self.getargvlist(name, default)[0] + + def getstring(self, name, default=None, replace=True): x = None - for s in [section] + self.fallbacksections: + for s in [self.section_name] + self.fallbacksections: try: x = self._cfg[s][name] break @@ -720,12 +817,12 @@ class IniReader: x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - self._subststack.append((section, name)) + self._subststack.append((self.section_name, name)) try: x = self._replace(x) finally: - assert self._subststack.pop() == (section, name) - # print "getdefault", section, name, "returned", repr(x) + assert self._subststack.pop() == (self.section_name, name) + # print "getstring", self.section_name, name, "returned", repr(x) return x def _apply_factors(self, s): @@ -821,8 +918,80 @@ class IniReader: return RE_ITEM_REF.sub(self._replace_match, x) return x - def _parse_command(self, command): - pass + +class _ArgvlistReader: + @classmethod + def getargvlist(cls, reader, section_val): + """Parse ``commands`` argvlist multiline string. + + :param str name: Key name in a section. + :param str section_val: Content stored by key. + + :rtype: list[list[str]] + :raise :class:`tox.exception.ConfigError`: + line-continuation ends nowhere while resolving for specified section + """ + commands = [] + current_command = "" + for line in section_val.splitlines(): + line = line.rstrip() + i = line.find("#") + if i != -1: + line = line[:i].rstrip() + if not line: + continue + if line.endswith("\\"): + current_command += " " + line[:-1] + continue + current_command += line + + if is_section_substitution(current_command): + replaced = reader._replace(current_command) + commands.extend(cls.getargvlist(reader, replaced)) + else: + commands.append(cls.processcommand(reader, current_command)) + current_command = "" + else: + if current_command: + raise tox.exception.ConfigError( + "line-continuation ends nowhere while resolving for [%s] %s" % + (reader.section_name, "commands")) + return commands + + @classmethod + def processcommand(cls, reader, command): + posargs = getattr(reader, "posargs", None) + + # Iterate through each word of the command substituting as + # appropriate to construct the new command string. This + # string is then broken up into exec argv components using + # shlex. + newcommand = "" + for word in CommandParser(command).words(): + if word == "{posargs}" or word == "[]": + if posargs: + newcommand += " ".join(posargs) + continue + elif word.startswith("{posargs:") and word.endswith("}"): + if posargs: + newcommand += " ".join(posargs) + continue + else: + word = word[9:-1] + new_arg = "" + new_word = reader._replace(word) + new_word = reader._replace(new_word) + new_arg += new_word + newcommand += new_arg + + # Construct shlex object that will not escape any values, + # use all values as is in argv. + shlexer = shlex.shlex(newcommand, posix=True) + shlexer.whitespace_split = True + shlexer.escape = '' + shlexer.commenters = '' + argv = list(shlexer) + return argv class CommandParser(object): diff --git a/tox/_venv.py b/tox/_venv.py index 0e01953..d4852bc 100644 --- a/tox/_venv.py +++ b/tox/_venv.py @@ -10,17 +10,17 @@ from tox._config import DepConfig class CreationConfig: def __init__(self, md5, python, version, sitepackages, - develop, deps): + usedevelop, deps): self.md5 = md5 self.python = python self.version = version self.sitepackages = sitepackages - self.develop = develop + self.usedevelop = usedevelop self.deps = deps def writeconfig(self, path): lines = ["%s %s" % (self.md5, self.python)] - lines.append("%s %d %d" % (self.version, self.sitepackages, self.develop)) + lines.append("%s %d %d" % (self.version, self.sitepackages, self.usedevelop)) for dep in self.deps: lines.append("%s %s" % dep) path.ensure() @@ -32,14 +32,14 @@ class CreationConfig: lines = path.readlines(cr=0) value = lines.pop(0).split(None, 1) md5, python = value - version, sitepackages, develop = lines.pop(0).split(None, 3) + version, sitepackages, usedevelop = lines.pop(0).split(None, 3) sitepackages = bool(int(sitepackages)) - develop = bool(int(develop)) + usedevelop = bool(int(usedevelop)) deps = [] for line in lines: md5, depstring = line.split(None, 1) deps.append((md5, depstring)) - return CreationConfig(md5, python, version, sitepackages, develop, deps) + return CreationConfig(md5, python, version, sitepackages, usedevelop, deps) except Exception: return None @@ -48,7 +48,7 @@ class CreationConfig: and self.python == other.python and self.version == other.version and self.sitepackages == other.sitepackages - and self.develop == other.develop + and self.usedevelop == other.usedevelop and self.deps == other.deps) @@ -143,11 +143,11 @@ class VirtualEnv(object): self.envconfig.deps, v) def _getliveconfig(self): - python = self.envconfig._basepython_info.executable + python = self.envconfig.python_info.executable md5 = getdigest(python) version = tox.__version__ sitepackages = self.envconfig.sitepackages - develop = self.envconfig.develop + develop = self.envconfig.usedevelop deps = [] for dep in self._getresolvedeps(): raw_dep = dep.name @@ -321,11 +321,13 @@ class VirtualEnv(object): for envname in self.envconfig.passenv: if envname in os.environ: env[envname] = os.environ[envname] - setenv = self.envconfig.setenv - if setenv: - env.update(setenv) + + env.update(self.envconfig.setenv) + env['VIRTUAL_ENV'] = str(self.path) + env.update(extraenv) + return env def test(self, redirect=False): diff --git a/tox/hookspecs.py b/tox/hookspecs.py new file mode 100644 index 0000000..103bda5 --- /dev/null +++ b/tox/hookspecs.py @@ -0,0 +1,32 @@ +""" Hook specifications for tox. + +""" + +from pluggy import HookspecMarker, HookimplMarker + +hookspec = HookspecMarker("tox") +hookimpl = HookimplMarker("tox") + + +@hookspec +def tox_addoption(parser): + """ add command line options to the argparse-style parser object.""" + + +@hookspec +def tox_configure(config): + """ called after command line options have been parsed and the ini-file has + been read. Please be aware that the config object layout may change as its + API was not designed yet wrt to providing stability (it was an internal + thing purely before tox-2.0). """ + + +@hookspec(firstresult=True) +def tox_get_python_executable(envconfig): + """ return a python executable for the given python base name. + The first plugin/hook which returns an executable path will determine it. + + ``envconfig`` is the testenv configuration which contains + per-testenv configuration, notably the ``.envname`` and ``.basepython`` + setting. + """ diff --git a/tox/interpreters.py b/tox/interpreters.py index 76075b8..66e6409 100644 --- a/tox/interpreters.py +++ b/tox/interpreters.py @@ -2,32 +2,31 @@ import sys import py import re import inspect +from tox import hookimpl class Interpreters: - def __init__(self): + def __init__(self, hook): self.name2executable = {} self.executable2info = {} + self.hook = hook - def get_executable(self, name): + def get_executable(self, envconfig): """ return path object to the executable for the given name (e.g. python2.6, python2.7, python etc.) if name is already an existing path, return name. If an interpreter cannot be found, return None. """ try: - return self.name2executable[name] + return self.name2executable[envconfig.envname] except KeyError: - self.name2executable[name] = e = find_executable(name) - return e - - def get_info(self, name=None, executable=None): - if name is None and executable is None: - raise ValueError("need to specify name or executable") - if name: - if executable is not None: - raise ValueError("cannot specify both name, executable") - executable = self.get_executable(name) + exe = self.hook.tox_get_python_executable(envconfig=envconfig) + self.name2executable[envconfig.envname] = exe + return exe + + def get_info(self, envconfig): + executable = self.get_executable(envconfig) + name = envconfig.basepython if not executable: return NoInterpreterInfo(name=name) try: @@ -125,31 +124,14 @@ class NoInterpreterInfo: return "<executable not found for: %s>" % self.name if sys.platform != "win32": - def find_executable(name): - return py.path.local.sysfind(name) + @hookimpl + def tox_get_python_executable(envconfig): + return py.path.local.sysfind(envconfig.basepython) else: - # Exceptions to the usual windows mapping - win32map = { - 'python': sys.executable, - 'jython': "c:\jython2.5.1\jython.bat", - } - - def locate_via_py(v_maj, v_min): - ver = "-%s.%s" % (v_maj, v_min) - script = "import sys; print(sys.executable)" - py_exe = py.path.local.sysfind('py') - if py_exe: - try: - exe = py_exe.sysexec(ver, '-c', script).strip() - except py.process.cmdexec.Error: - exe = None - if exe: - exe = py.path.local(exe) - if exe.check(): - return exe - - def find_executable(name): + @hookimpl + def tox_get_python_executable(envconfig): + name = envconfig.basepython p = py.path.local.sysfind(name) if p: return p @@ -170,6 +152,26 @@ else: if m: return locate_via_py(*m.groups()) + # Exceptions to the usual windows mapping + win32map = { + 'python': sys.executable, + 'jython': "c:\jython2.5.1\jython.bat", + } + + def locate_via_py(v_maj, v_min): + ver = "-%s.%s" % (v_maj, v_min) + script = "import sys; print(sys.executable)" + py_exe = py.path.local.sysfind('py') + if py_exe: + try: + exe = py_exe.sysexec(ver, '-c', script).strip() + except py.process.cmdexec.Error: + exe = None + if exe: + exe = py.path.local(exe) + if exe.check(): + return exe + def pyinfo(): import sys |