diff options
| author | holger krekel <holger@merlinux.eu> | 2015-05-11 12:06:39 +0200 |
|---|---|---|
| committer | holger krekel <holger@merlinux.eu> | 2015-05-11 12:06:39 +0200 |
| commit | 097438b8a986bcba6820f930d875dc2b4b1e3bb1 (patch) | |
| tree | fe743836073316c19ef08c347aa10e697753931f | |
| parent | 0a29364f4890691dc1979f694fc924c441ac65e9 (diff) | |
| download | tox-097438b8a986bcba6820f930d875dc2b4b1e3bb1.tar.gz | |
refactor testenv section parser to work by registering ini attributes
at tox_addoption() time. Introduce new "--help-ini" or "--hi" option
to show all testenv variables.
| -rw-r--r-- | CHANGELOG | 4 | ||||
| -rw-r--r-- | tests/test_config.py | 183 | ||||
| -rw-r--r-- | tests/test_venv.py | 2 | ||||
| -rw-r--r-- | tox.ini | 9 | ||||
| -rw-r--r-- | tox/_cmdline.py | 26 | ||||
| -rw-r--r-- | tox/_config.py | 662 | ||||
| -rw-r--r-- | tox/_venv.py | 24 |
7 files changed, 542 insertions, 368 deletions
@@ -36,6 +36,10 @@ based on pytest's externalized "pluggy" system. See tox/hookspecs.py for the current hooks. +- introduce parser.add_testenv_attribute() to register an ini-variable + for testenv sections. Can be used from plugins through the + tox_add_option hook. + 1.9.2 ----------- diff --git a/tests/test_config.py b/tests/test_config.py index 522a22f..a963f44 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,7 +18,7 @@ class TestVenvConfig: assert config.toxworkdir.realpath() == tmpdir.join(".tox").realpath() assert config.envconfigs['py1'].basepython == sys.executable assert config.envconfigs['py1'].deps == [] - assert not config.envconfigs['py1'].platform + assert config.envconfigs['py1'].platform == ".*" def test_config_parsing_multienv(self, tmpdir, newconfig): config = newconfig([], """ @@ -92,12 +92,12 @@ class TestVenvConfig: """ Ensure correct parseini._is_same_dep is working with a few samples. """ - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') - assert not parseini._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') + assert not DepOption._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') class TestConfigPlatform: @@ -219,8 +219,8 @@ class TestIniParserAgainstCommandsKey: commands = echo {[section]key} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [["echo", "whatever"]] def test_command_substitution_from_other_section_multiline(self, newconfig): @@ -244,8 +244,8 @@ class TestIniParserAgainstCommandsKey: # comment is omitted echo {[base]commands} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [ "cmd1 param11 param12".split(), "cmd2 param21 param22".split(), @@ -256,16 +256,16 @@ class TestIniParserAgainstCommandsKey: class TestIniParser: - def test_getdefault_single(self, tmpdir, newconfig): + def test_getstring_single(self, tmpdir, newconfig): config = newconfig(""" [section] key=value """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key") assert x == "value" - assert not reader.getdefault("section", "hello") - x = reader.getdefault("section", "hello", "world") + assert not reader.getstring("hello") + x = reader.getstring("hello", "world") assert x == "world" def test_missing_substitution(self, tmpdir, newconfig): @@ -273,40 +273,40 @@ class TestIniParser: [mydefault] key2={xyz} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("mydefault", config._cfg, fallbacksections=['mydefault']) assert reader is not None - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("mydefault", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_fallback_sections(self, tmpdir, newconfig): + def test_getstring_fallback_sections(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2=value2 [section] key=value """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) - x = reader.getdefault("section", "key2") + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) + x = reader.getstring("key2") assert x == "value2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "world") + x = reader.getstring("key3", "world") assert x == "world" - def test_getdefault_substitution(self, tmpdir, newconfig): + def test_getstring_substitution(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2={value2} [section] key={value} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) reader.addsubstitutions(value="newvalue", value2="newvalue2") - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "newvalue2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "{value2}") + x = reader.getstring("key3", "{value2}") assert x == "newvalue2" def test_getlist(self, tmpdir, newconfig): @@ -316,9 +316,9 @@ class TestIniParser: item1 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getlist("section", "key2") + x = reader.getlist("key2") assert x == ['item1', 'grr'] def test_getdict(self, tmpdir, newconfig): @@ -328,28 +328,31 @@ class TestIniParser: key1=item1 key2={item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getdict("section", "key2") + x = reader.getdict("key2") assert 'key1' in x assert 'key2' in x assert x['key1'] == 'item1' assert x['key2'] == 'grr' - def test_getdefault_environment_substitution(self, monkeypatch, newconfig): + x = reader.getdict("key3", {1: 2}) + assert x == {1: 2} + + def test_getstring_environment_substitution(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] key1={env:KEY1} key2={env:KEY2} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("section", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_environment_substitution_with_default(self, monkeypatch, newconfig): + def test_getstring_environment_substitution_with_default(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] @@ -357,12 +360,12 @@ class TestIniParser: key2={env:KEY2:DEFAULT_VALUE} key3={env:KEY3:} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "DEFAULT_VALUE" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert x == "" def test_value_matches_section_substituion(self): @@ -373,15 +376,15 @@ class TestIniParser: assert is_section_substitution("{[setup]}") is None assert is_section_substitution("{[setup] commands}") is None - def test_getdefault_other_section_substitution(self, newconfig): + def test_getstring_other_section_substitution(self, newconfig): config = newconfig(""" [section] key = rue [testenv] key = t{[section]key} """) - reader = IniReader(config._cfg) - x = reader.getdefault("testenv", "key") + reader = SectionReader("testenv", config._cfg) + x = reader.getstring("key") assert x == "true" def test_argvlist(self, tmpdir, newconfig): @@ -391,12 +394,12 @@ class TestIniParser: cmd1 {item1} {item2} cmd2 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"], ["cmd2", "grr"]] @@ -405,9 +408,9 @@ class TestIniParser: [section] comm = py.test {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([r"hello\this"]) - argv = reader.getargv("section", "comm") + argv = reader.getargv("comm") assert argv == ["py.test", "hello\\this"] def test_argvlist_multiline(self, tmpdir, newconfig): @@ -417,12 +420,12 @@ class TestIniParser: cmd1 {item1} \ # a comment {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"]] def test_argvlist_quoting_in_command(self, tmpdir, newconfig): @@ -432,8 +435,8 @@ class TestIniParser: cmd1 'with space' \ # a comment 'after the comment' """) - reader = IniReader(config._cfg) - x = reader.getargvlist("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getargvlist("key1") assert x == [["cmd1", "with space", "after the comment"]] def test_argvlist_positional_substitution(self, tmpdir, newconfig): @@ -444,22 +447,22 @@ class TestIniParser: cmd2 {posargs:{item2} \ other} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] + posargs assert argvlist[1] == ["cmd2"] + posargs - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([], item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] assert argvlist[1] == ["cmd2", "value2", "other"] @@ -471,10 +474,10 @@ class TestIniParser: cmd2 -f '{posargs}' cmd3 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "bar"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "--foo-args=foo bar"], ["cmd2", "-f", "foo bar"], ["cmd3", "-f", "foo", "bar"]] @@ -485,10 +488,10 @@ class TestIniParser: key2= cmd1 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "'bar", "baz'"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "-f", "foo", "bar baz"]] def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig): @@ -500,11 +503,11 @@ class TestIniParser: cmd2 -m '\'something\'' [] cmd3 something[]else """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs) - argvlist = reader.getargvlist('section', 'key') + argvlist = reader.getargvlist('key') assert argvlist[0] == ['cmd0'] + posargs assert argvlist[1] == ['cmd1', '-m', '[abc]'] assert argvlist[2] == ['cmd2', '-m', "something"] + posargs @@ -516,32 +519,32 @@ class TestIniParser: key = py.test -n5 --junitxml={envlogdir}/junit-{envname}.xml [] """ config = newconfig(inisource) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, envlogdir='ENV_LOG_DIR', envname='ENV_NAME') expected = [ 'py.test', '-n5', '--junitxml=ENV_LOG_DIR/junit-ENV_NAME.xml', 'hello', 'world' ] - assert reader.getargvlist('section', 'key')[0] == expected + assert reader.getargvlist('key')[0] == expected def test_getargv(self, newconfig): config = newconfig(""" [section] key=some command "with quoting" """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) expected = ['some', 'command', 'with quoting'] - assert reader.getargv('section', 'key') == expected + assert reader.getargv('key') == expected def test_getpath(self, tmpdir, newconfig): config = newconfig(""" [section] path1={HELLO} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(toxinidir=tmpdir, HELLO="mypath") - x = reader.getpath("section", "path1", tmpdir) + x = reader.getpath("path1", tmpdir) assert x == tmpdir.join("mypath") def test_getbool(self, tmpdir, newconfig): @@ -553,13 +556,13 @@ class TestIniParser: key2a=falsE key5=yes """) - reader = IniReader(config._cfg) - assert reader.getbool("section", "key1") is True - assert reader.getbool("section", "key1a") is True - assert reader.getbool("section", "key2") is False - assert reader.getbool("section", "key2a") is False - py.test.raises(KeyError, 'reader.getbool("section", "key3")') - py.test.raises(tox.exception.ConfigError, 'reader.getbool("section", "key5")') + reader = SectionReader("section", config._cfg) + assert reader.getbool("key1") is True + assert reader.getbool("key1a") is True + assert reader.getbool("key2") is False + assert reader.getbool("key2a") is False + py.test.raises(KeyError, 'reader.getbool("key3")') + py.test.raises(tox.exception.ConfigError, 'reader.getbool("key5")') class TestConfigTestEnv: @@ -585,7 +588,7 @@ class TestConfigTestEnv: assert envconfig.commands == [["xyz", "--abc"]] assert envconfig.changedir == config.setupdir assert envconfig.sitepackages is False - assert envconfig.develop is False + assert envconfig.usedevelop is False assert envconfig.envlogdir == envconfig.envdir.join("log") assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] hashseed = envconfig.setenv['PYTHONHASHSEED'] @@ -605,7 +608,7 @@ class TestConfigTestEnv: [testenv] usedevelop = True """) - assert not config.envconfigs["python"].develop + assert not config.envconfigs["python"].usedevelop def test_specific_command_overrides(self, tmpdir, newconfig): config = newconfig(""" @@ -1371,7 +1374,7 @@ class TestHashseedOption: def test_noset(self, tmpdir, newconfig): args = ['--hashseed', 'noset'] envconfig = self._get_envconfig(newconfig, args=args) - assert envconfig.setenv is None + assert envconfig.setenv == {} def test_noset_with_setenv(self, tmpdir, newconfig): tox_ini = """ diff --git a/tests/test_venv.py b/tests/test_venv.py index bd6926e..b32affd 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -464,7 +464,7 @@ class TestCreationConfig: venv = VirtualEnv(envconfig, session=mocksession) venv.update() cconfig = venv._getliveconfig() - cconfig.develop = True + cconfig.usedevelop = True cconfig.writeconfig(venv.path_config) mocksession._clearmocks() venv.update() @@ -5,8 +5,10 @@ envlist=py27,py26,py34,py33,pypy,flakes commands=echo {posargs} [testenv] -commands=py.test --junitxml={envlogdir}/junit-{envname}.xml {posargs} +commands= py.test --timeout=60 {posargs} + deps=pytest>=2.3.5 + pytest-timeout [testenv:docs] basepython=python @@ -14,11 +16,10 @@ changedir=doc deps=sphinx {[testenv]deps} commands= - py.test -v \ - --junitxml={envlogdir}/junit-{envname}.xml \ - check_sphinx.py {posargs} + py.test -v check_sphinx.py {posargs} [testenv:flakes] +qwe = 123 deps = pytest-flakes>=0.2 pytest-pep8 diff --git a/tox/_cmdline.py b/tox/_cmdline.py index 3dd9582..342c2ad 100644 --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -25,12 +25,34 @@ def now(): def main(args=None): try: config = parseconfig(args) + if config.option.help: + show_help(config) + raise SystemExit(0) + elif config.option.helpini: + show_help_ini(config) + raise SystemExit(0) retcode = Session(config).runcommand() raise SystemExit(retcode) except KeyboardInterrupt: raise SystemExit(2) +def show_help(config): + tw = py.io.TerminalWriter() + tw.write(config._parser.format_help()) + tw.line() + + +def show_help_ini(config): + tw = py.io.TerminalWriter() + tw.sep("-", "per-testenv attributes") + for env_attr in config._testenv_attr: + tw.line("%-15s %-8s default: %s" % + (env_attr.name, "<" + env_attr.type + ">", env_attr.default), bold=True) + tw.line(env_attr.help) + tw.line() + + class Action(object): def __init__(self, session, venv, msg, args): self.venv = venv @@ -487,7 +509,7 @@ class Session: venv.status = "platform mismatch" continue # we simply omit non-matching platforms if self.setupenv(venv): - if venv.envconfig.develop: + if venv.envconfig.usedevelop: self.developpkg(venv, self.config.setupdir) elif self.config.skipsdist or venv.envconfig.skip_install: self.finishvenv(venv) @@ -566,7 +588,7 @@ class Session: self.report.line(" deps=%s" % envconfig.deps) self.report.line(" envdir= %s" % envconfig.envdir) self.report.line(" downloadcache=%s" % envconfig.downloadcache) - self.report.line(" usedevelop=%s" % envconfig.develop) + self.report.line(" usedevelop=%s" % envconfig.usedevelop) def showenvs(self): for env in self.config.envlist: diff --git a/tox/_config.py b/tox/_config.py index 6f3662c..9189808 100644 --- a/tox/_config.py +++ b/tox/_config.py @@ -38,6 +38,122 @@ def get_plugin_manager(): return pm +class MyParser: + def __init__(self): + self.argparser = argparse.ArgumentParser( + description="tox options", add_help=False) + self._testenv_attr = [] + + def add_argument(self, *args, **kwargs): + return self.argparser.add_argument(*args, **kwargs) + + def add_testenv_attribute(self, name, type, help, default=None, postprocess=None): + self._testenv_attr.append(VenvAttribute(name, type, default, help, postprocess)) + + def add_testenv_attribute_obj(self, obj): + assert hasattr(obj, "name") + assert hasattr(obj, "type") + assert hasattr(obj, "help") + assert hasattr(obj, "postprocess") + self._testenv_attr.append(obj) + + def parse_args(self, args): + return self.argparser.parse_args(args) + + def format_help(self): + return self.argparser.format_help() + + +class VenvAttribute: + def __init__(self, name, type, default, help, postprocess): + self.name = name + self.type = type + self.default = default + self.help = help + self.postprocess = postprocess + + +class DepOption: + name = "deps" + type = "line-list" + help = "each line specifies a dependency in pip/setuptools format." + default = () + + def postprocess(self, config, reader, section_val): + deps = [] + for depline in section_val: + m = re.match(r":(\w+):\s*(\S+)", depline) + if m: + iname, name = m.groups() + ixserver = config.indexserver[iname] + else: + name = depline.strip() + ixserver = None + name = self._replace_forced_dep(name, config) + deps.append(DepConfig(name, ixserver)) + return deps + + def _replace_forced_dep(self, name, config): + """ + Override the given dependency config name taking --force-dep-version + option into account. + + :param name: dep config, for example ["pkg==1.0", "other==2.0"]. + :param config: Config instance + :return: the new dependency that should be used for virtual environments + """ + if not config.option.force_dep: + return name + for forced_dep in config.option.force_dep: + if self._is_same_dep(forced_dep, name): + return forced_dep + return name + + @classmethod + def _is_same_dep(cls, dep1, dep2): + """ + Returns True if both dependency definitions refer to the + same package, even if versions differ. + """ + dep1_name = pkg_resources.Requirement.parse(dep1).project_name + dep2_name = pkg_resources.Requirement.parse(dep2).project_name + return dep1_name == dep2_name + + +class PosargsOption: + name = "args_are_paths" + type = "bool" + default = True + help = "treat positional args in commands as paths" + + def postprocess(self, config, reader, section_val): + args = config.option.args + if args: + if section_val: + args = [] + for arg in config.option.args: + if arg: + origpath = config.invocationcwd.join(arg, abs=True) + if origpath.check(): + arg = reader.getpath("changedir", ".").bestrelpath(origpath) + args.append(arg) + reader.addsubstitutions(args) + return section_val + + +class InstallcmdOption: + name = "install_command" + type = "argv" + default = "pip install {opts} {packages}" + help = "install command for dependencies and package under test." + + def postprocess(self, config, reader, section_val): + if '{packages}' not in section_val: + raise tox.exception.ConfigError( + "'install_command' must contain '{packages}' substitution") + return section_val + + def parseconfig(args=None): """ :param list[str] args: Optional list of arguments. @@ -52,13 +168,15 @@ def parseconfig(args=None): args = sys.argv[1:] # prepare command line options - parser = argparse.ArgumentParser(description=__doc__) + parser = MyParser() pm.hook.tox_addoption(parser=parser) # parse command line options option = parser.parse_args(args) interpreters = tox.interpreters.Interpreters(hook=pm.hook) config = Config(pluginmanager=pm, option=option, interpreters=interpreters) + config._parser = parser + config._testenv_attr = parser._testenv_attr # parse ini file basename = config.option.configfile @@ -111,6 +229,10 @@ def tox_addoption(parser): parser.add_argument("--version", nargs=0, action=VersionAction, dest="version", help="report version information to stdout.") + parser.add_argument("-h", "--help", action="store_true", dest="help", + help="show help about options") + parser.add_argument("--help-ini", "--hi", action="store_true", dest="helpini", + help="show help about ini-names") parser.add_argument("-v", nargs=0, action=CountAction, default=0, dest="verbosity", help="increase verbosity of reporting output.") @@ -156,6 +278,7 @@ def tox_addoption(parser): "all commands and results involved. This will turn off " "pass-through output from running test commands which is " "instead captured into the json result file.") + # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. parser.add_argument("--hashseed", action="store", metavar="SEED", default=None, @@ -175,7 +298,130 @@ def tox_addoption(parser): parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") - return parser + + # add various core venv interpreter attributes + + parser.add_testenv_attribute( + name="envdir", type="path", default="{toxworkdir}/{envname}", + help="venv directory") + + parser.add_testenv_attribute( + name="envtmpdir", type="path", default="{envdir}/tmp", + help="venv temporary directory") + + parser.add_testenv_attribute( + name="envlogdir", type="path", default="{envdir}/log", + help="venv log directory") + + def downloadcache(config, reader, section_val): + if section_val: + # env var, if present, takes precedence + downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", section_val) + return py.path.local(downloadcache) + + parser.add_testenv_attribute( + name="downloadcache", type="string", default=None, postprocess=downloadcache, + help="(deprecated) set PIP_DOWNLOAD_CACHE.") + + parser.add_testenv_attribute( + name="changedir", type="path", default="{toxinidir}", + help="directory to change to when running commands") + + parser.add_testenv_attribute_obj(PosargsOption()) + + parser.add_testenv_attribute( + name="skip_install", type="bool", default=False, + help="Do not install the current package. This can be used when " + "you need the virtualenv management but do not want to install " + "the current package") + + def recreate(config, reader, section_val): + if config.option.recreate: + return True + return section_val + + parser.add_testenv_attribute( + name="recreate", type="bool", default=False, postprocess=recreate, + help="always recreate this test environment.") + + def setenv(config, reader, section_val): + setenv = section_val + if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: + setenv['PYTHONHASHSEED'] = config.hashseed + return setenv + + parser.add_testenv_attribute( + name="setenv", type="dict", postprocess=setenv, + help="list of X=Y lines with environment variable settings") + + def passenv(config, reader, section_val): + passenv = set(["PATH"]) + if sys.platform == "win32": + passenv.add("SYSTEMROOT") # needed for python's crypto module + passenv.add("PATHEXT") # needed for discovering executables + for spec in section_val: + for name in os.environ: + if fnmatchcase(name.upper(), spec.upper()): + passenv.add(name) + return passenv + + parser.add_testenv_attribute( + name="passenv", type="space-separated-list", postprocess=passenv, + help="environment variables names which shall be passed " + "from tox invocation to test environment when executing commands.") + + parser.add_testenv_attribute( + name="whitelist_externals", type="line-list", + help="each lines specifies a path or basename for which tox will not warn " + "about it coming from outside the test environment.") + + parser.add_testenv_attribute( + name="platform", type="string", default=".*", + help="regular expression which must match against ``sys.platform``. " + "otherwise testenv will be skipped.") + + def sitepackages(config, reader, section_val): + return config.option.sitepackages or section_val + + parser.add_testenv_attribute( + name="sitepackages", type="bool", default=False, postprocess=sitepackages, + help="Set to ``True`` if you want to create virtual environments that also " + "have access to globally installed packages.") + + def pip_pre(config, reader, section_val): + return config.option.pre or section_val + + parser.add_testenv_attribute( + name="pip_pre", type="bool", default=False, postprocess=pip_pre, + help="If ``True``, adds ``--pre`` to the ``opts`` passed to " + "the install command. ") + + def develop(config, reader, section_val): + return not config.option.installpkg and (section_val or config.option.develop) + + parser.add_testenv_attribute( + name="usedevelop", type="bool", postprocess=develop, default=False, + help="install package in develop/editable mode") + + def basepython_default(config, reader, section_val): + if section_val is None: + for f in reader.factors: + if f in default_factors: + return default_factors[f] + return sys.executable + return str(section_val) + + parser.add_testenv_attribute( + name="basepython", type="string", default=None, postprocess=basepython_default, + help="executable name or path of interpreter used to create a " + "virtual test environment.") + + parser.add_testenv_attribute_obj(InstallcmdOption()) + parser.add_testenv_attribute_obj(DepOption()) + + parser.add_testenv_attribute( + name="commands", type="argvlist", default="", + help="each line specifies a test command and can use substitution.") class Config(object): @@ -272,12 +518,10 @@ class parseini: self.config = config ctxname = getcontextname() if ctxname == "jenkins": - reader = IniReader(self._cfg, fallbacksections=['tox']) - toxsection = "tox:%s" % ctxname + reader = SectionReader("tox:jenkins", self._cfg, fallbacksections=['tox']) distshare_default = "{toxworkdir}/distshare" elif not ctxname: - reader = IniReader(self._cfg) - toxsection = "tox" + reader = SectionReader("tox", self._cfg) distshare_default = "{homedir}/.tox/distshare" else: raise ValueError("invalid context") @@ -292,18 +536,17 @@ class parseini: reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) - config.toxworkdir = reader.getpath(toxsection, "toxworkdir", - "{toxinidir}/.tox") - config.minversion = reader.getdefault(toxsection, "minversion", None) + config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox") + config.minversion = reader.getstring("minversion", None) if not config.option.skip_missing_interpreters: config.option.skip_missing_interpreters = \ - reader.getbool(toxsection, "skip_missing_interpreters", False) + reader.getbool("skip_missing_interpreters", False) # determine indexserver dictionary config.indexserver = {'default': IndexServerConfig('default')} prefix = "indexserver" - for line in reader.getlist(toxsection, prefix): + for line in reader.getlist(prefix): name, url = map(lambda x: x.strip(), line.split("=", 1)) config.indexserver[name] = IndexServerConfig(name, url) @@ -328,16 +571,15 @@ class parseini: config.indexserver[name] = IndexServerConfig(name, override) reader.addsubstitutions(toxworkdir=config.toxworkdir) - config.distdir = reader.getpath(toxsection, "distdir", "{toxworkdir}/dist") + config.distdir = reader.getpath("distdir", "{toxworkdir}/dist") reader.addsubstitutions(distdir=config.distdir) - config.distshare = reader.getpath(toxsection, "distshare", - distshare_default) + config.distshare = reader.getpath("distshare", distshare_default) reader.addsubstitutions(distshare=config.distshare) - config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None) - config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}") + config.sdistsrc = reader.getpath("sdistsrc", None) + config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") - config.envlist, all_envs = self._getenvdata(reader, toxsection) + config.envlist, all_envs = self._getenvdata(reader) # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -345,7 +587,7 @@ class parseini: known_factors.add("python") # factors stated in config envlist - stated_envlist = reader.getdefault(toxsection, "envlist", replace=False) + stated_envlist = reader.getstring("envlist", replace=False) if stated_envlist: for env in _split_env(stated_envlist): known_factors.update(env.split('-')) @@ -356,13 +598,13 @@ class parseini: factors = set(name.split('-')) if section in self._cfg or factors <= known_factors: config.envconfigs[name] = \ - self._makeenvconfig(name, section, reader._subs, config) + self.make_envconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs - and config.envconfigs[name].develop + and config.envconfigs[name].usedevelop for name in config.envlist) - config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop) + config.skipsdist = reader.getbool("skipsdist", all_develop) def _list_section_factors(self, section): factors = set() @@ -372,115 +614,51 @@ class parseini: factors.update(*mapcat(_split_factor_expr, exprs)) return factors - def _makeenvconfig(self, name, section, subs, config): + def make_envconfig(self, name, section, subs, config): vc = VenvConfig(config=config, envname=name) factors = set(name.split('-')) - reader = IniReader(self._cfg, fallbacksections=["testenv"], factors=factors) + reader = SectionReader(section, self._cfg, fallbacksections=["testenv"], + factors=factors) reader.addsubstitutions(**subs) - vc.develop = ( - not config.option.installpkg - and reader.getbool(section, "usedevelop", config.option.develop)) - vc.envdir = reader.getpath(section, "envdir", "{toxworkdir}/%s" % name) - vc.args_are_paths = reader.getbool(section, "args_are_paths", True) - if reader.getdefault(section, "python", None): - raise tox.exception.ConfigError( - "'python=' key was renamed to 'basepython='") - bp = next((default_factors[f] for f in factors if f in default_factors), - sys.executable) - vc.basepython = reader.getdefault(section, "basepython", bp) - - reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname, - envbindir=vc.envbindir, envpython=vc.envpython, - envsitepackagesdir=vc.envsitepackagesdir) - vc.envtmpdir = reader.getpath(section, "tmpdir", "{envdir}/tmp") - vc.envlogdir = reader.getpath(section, "envlogdir", "{envdir}/log") - reader.addsubstitutions(envlogdir=vc.envlogdir, envtmpdir=vc.envtmpdir) - vc.changedir = reader.getpath(section, "changedir", "{toxinidir}") - if config.option.recreate: - vc.recreate = True - else: - vc.recreate = reader.getbool(section, "recreate", False) - args = config.option.args - if args: - if vc.args_are_paths: - args = [] - for arg in config.option.args: - if arg: - origpath = config.invocationcwd.join(arg, abs=True) - if origpath.check(): - arg = vc.changedir.bestrelpath(origpath) - args.append(arg) - reader.addsubstitutions(args) - setenv = {} - if config.hashseed is not None: - setenv['PYTHONHASHSEED'] = config.hashseed - setenv.update(reader.getdict(section, 'setenv')) - - # read passenv - vc.passenv = set(["PATH"]) - if sys.platform == "win32": - vc.passenv.add("SYSTEMROOT") # needed for python's crypto module - vc.passenv.add("PATHEXT") # needed for discovering executables - for spec in reader.getlist(section, "passenv", sep=" "): - for name in os.environ: - if fnmatchcase(name.lower(), spec.lower()): - vc.passenv.add(name) - - vc.setenv = setenv - if not vc.setenv: - vc.setenv = None - - vc.commands = reader.getargvlist(section, "commands") - vc.whitelist_externals = reader.getlist(section, - "whitelist_externals") - vc.deps = [] - for depline in reader.getlist(section, "deps"): - m = re.match(r":(\w+):\s*(\S+)", depline) - if m: - iname, name = m.groups() - ixserver = config.indexserver[iname] + reader.addsubstitutions(envname=name) + + for env_attr in config._testenv_attr: + atype = env_attr.type + if atype in ("bool", "path", "string", "dict", "argv", "argvlist"): + meth = getattr(reader, "get" + atype) + res = meth(env_attr.name, env_attr.default) + elif atype == "space-separated-list": + res = reader.getlist(env_attr.name, sep=" ") + elif atype == "line-list": + res = reader.getlist(env_attr.name, sep="\n") else: - name = depline.strip() - ixserver = None - name = self._replace_forced_dep(name, config) - vc.deps.append(DepConfig(name, ixserver)) - - platform = "" - for platform in reader.getlist(section, "platform"): - if platform.strip(): - break - vc.platform = platform - - vc.sitepackages = ( - self.config.option.sitepackages - or reader.getbool(section, "sitepackages", False)) - - vc.downloadcache = None - downloadcache = reader.getdefault(section, "downloadcache") - if downloadcache: - # env var, if present, takes precedence - downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", downloadcache) - vc.downloadcache = py.path.local(downloadcache) - - vc.install_command = reader.getargv( - section, - "install_command", - "pip install {opts} {packages}", - ) - if '{packages}' not in vc.install_command: - raise tox.exception.ConfigError( - "'install_command' must contain '{packages}' substitution") - vc.pip_pre = config.option.pre or reader.getbool( - section, "pip_pre", False) - - vc.skip_install = reader.getbool(section, "skip_install", False) - + raise ValueError("unknown type %r" % (atype,)) + + if env_attr.postprocess: + res = env_attr.postprocess(config, reader, res) + setattr(vc, env_attr.name, res) + + if atype == "path": + reader.addsubstitutions(**{env_attr.name: res}) + + if env_attr.name == "install_command": + reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, + envsitepackagesdir=vc.envsitepackagesdir) + + # XXX introduce some testenv verification like this: + # try: + # sec = self._cfg[section] + # except KeyError: + # sec = self._cfg["testenv"] + # for name in sec: + # if name not in names: + # print ("unknown testenv attribute: %r" % (name,)) return vc - def _getenvdata(self, reader, toxsection): + def _getenvdata(self, reader): envstr = self.config.option.env \ or os.environ.get("TOXENV") \ - or reader.getdefault(toxsection, "envlist", replace=False) \ + or reader.getstring("envlist", replace=False) \ or [] envlist = _split_env(envstr) @@ -497,32 +675,6 @@ class parseini: return envlist, all_envs - def _replace_forced_dep(self, name, config): - """ - Override the given dependency config name taking --force-dep-version - option into account. - - :param name: dep config, for example ["pkg==1.0", "other==2.0"]. - :param config: Config instance - :return: the new dependency that should be used for virtual environments - """ - if not config.option.force_dep: - return name - for forced_dep in config.option.force_dep: - if self._is_same_dep(forced_dep, name): - return forced_dep - return name - - @classmethod - def _is_same_dep(cls, dep1, dep2): - """ - Returns True if both dependency definitions refer to the - same package, even if versions differ. - """ - dep1_name = pkg_resources.Requirement.parse(dep1).project_name - dep2_name = pkg_resources.Requirement.parse(dep2).project_name - return dep1_name == dep2_name - def _split_env(env): """if handed a list, action="append" was used for -e """ @@ -589,8 +741,9 @@ RE_ITEM_REF = re.compile( re.VERBOSE) -class IniReader: - def __init__(self, cfgparser, fallbacksections=None, factors=()): +class SectionReader: + def __init__(self, section_name, cfgparser, fallbacksections=None, factors=()): + self.section_name = section_name self._cfg = cfgparser self.fallbacksections = fallbacksections or [] self.factors = factors @@ -602,129 +755,39 @@ class IniReader: if _posargs: self.posargs = _posargs - def getpath(self, section, name, defaultpath): + def getpath(self, name, defaultpath): toxinidir = self._subs['toxinidir'] - path = self.getdefault(section, name, defaultpath) + path = self.getstring(name, defaultpath) if path is None: return path return toxinidir.join(path, abs=True) - def getlist(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getlist(self, name, sep="\n"): + s = self.getstring(name, None) if s is None: return [] return [x.strip() for x in s.split(sep) if x.strip()] - def getdict(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getdict(self, name, default=None, sep="\n"): + s = self.getstring(name, None) if s is None: - return {} + return default or {} value = {} for line in s.split(sep): - if not line.strip(): - continue - name, rest = line.split('=', 1) - value[name.strip()] = rest.strip() + if line.strip(): + name, rest = line.split('=', 1) + value[name.strip()] = rest.strip() return value - def getargvlist(self, section, name): - """Get arguments for every parsed command. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - content = self.getdefault(section, name, '', replace=False) - return self._parse_commands(section, name, content) - - def _parse_commands(self, section, name, content): - """Parse commands from key content in specified section. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :param str content: Content stored by key. - - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - commands = [] - current_command = "" - for line in content.splitlines(): - line = line.rstrip() - i = line.find("#") - if i != -1: - line = line[:i].rstrip() - if not line: - continue - if line.endswith("\\"): - current_command += " " + line[:-1] - continue - current_command += line - - if is_section_substitution(current_command): - replaced = self._replace(current_command) - commands.extend(self._parse_commands(section, name, replaced)) - else: - commands.append(self._processcommand(current_command)) - current_command = "" - else: - if current_command: - raise tox.exception.ConfigError( - "line-continuation ends nowhere while resolving for [%s] %s" % - (section, name)) - return commands - - def _processcommand(self, command): - posargs = getattr(self, "posargs", None) - - # Iterate through each word of the command substituting as - # appropriate to construct the new command string. This - # string is then broken up into exec argv components using - # shlex. - newcommand = "" - for word in CommandParser(command).words(): - if word == "{posargs}" or word == "[]": - if posargs: - newcommand += " ".join(posargs) - continue - elif word.startswith("{posargs:") and word.endswith("}"): - if posargs: - newcommand += " ".join(posargs) - continue - else: - word = word[9:-1] - new_arg = "" - new_word = self._replace(word) - new_word = self._replace(new_word) - new_arg += new_word - newcommand += new_arg - - # Construct shlex object that will not escape any values, - # use all values as is in argv. - shlexer = shlex.shlex(newcommand, posix=True) - shlexer.whitespace_split = True - shlexer.escape = '' - shlexer.commenters = '' - argv = list(shlexer) - return argv - - def getargv(self, section, name, default=None, replace=True): - command = self.getdefault( - section, name, default=default, replace=False) - return self._processcommand(command.strip()) - - def getbool(self, section, name, default=None): - s = self.getdefault(section, name, default) + def getbool(self, name, default=None): + s = self.getstring(name, default) if not s: s = default if s is None: raise KeyError("no config value [%s] %s found" % ( - section, name)) + self.section_name, name)) if not isinstance(s, bool): if s.lower() == "true": @@ -736,9 +799,16 @@ class IniReader: "boolean value %r needs to be 'True' or 'False'") return s - def getdefault(self, section, name, default=None, replace=True): + def getargvlist(self, name, default=""): + s = self.getstring(name, default, replace=False) + return _ArgvlistReader.getargvlist(self, s) + + def getargv(self, name, default=""): + return self.getargvlist(name, default)[0] + + def getstring(self, name, default=None, replace=True): x = None - for s in [section] + self.fallbacksections: + for s in [self.section_name] + self.fallbacksections: try: x = self._cfg[s][name] break @@ -751,12 +821,12 @@ class IniReader: x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - self._subststack.append((section, name)) + self._subststack.append((self.section_name, name)) try: x = self._replace(x) finally: - assert self._subststack.pop() == (section, name) - # print "getdefault", section, name, "returned", repr(x) + assert self._subststack.pop() == (self.section_name, name) + # print "getstring", self.section_name, name, "returned", repr(x) return x def _apply_factors(self, s): @@ -852,8 +922,80 @@ class IniReader: return RE_ITEM_REF.sub(self._replace_match, x) return x - def _parse_command(self, command): - pass + +class _ArgvlistReader: + @classmethod + def getargvlist(cls, reader, section_val): + """Parse ``commands`` argvlist multiline string. + + :param str name: Key name in a section. + :param str section_val: Content stored by key. + + :rtype: list[list[str]] + :raise :class:`tox.exception.ConfigError`: + line-continuation ends nowhere while resolving for specified section + """ + commands = [] + current_command = "" + for line in section_val.splitlines(): + line = line.rstrip() + i = line.find("#") + if i != -1: + line = line[:i].rstrip() + if not line: + continue + if line.endswith("\\"): + current_command += " " + line[:-1] + continue + current_command += line + + if is_section_substitution(current_command): + replaced = reader._replace(current_command) + commands.extend(cls.getargvlist(reader, replaced)) + else: + commands.append(cls.processcommand(reader, current_command)) + current_command = "" + else: + if current_command: + raise tox.exception.ConfigError( + "line-continuation ends nowhere while resolving for [%s] %s" % + (reader.section_name, "commands")) + return commands + + @classmethod + def processcommand(cls, reader, command): + posargs = getattr(reader, "posargs", None) + + # Iterate through each word of the command substituting as + # appropriate to construct the new command string. This + # string is then broken up into exec argv components using + # shlex. + newcommand = "" + for word in CommandParser(command).words(): + if word == "{posargs}" or word == "[]": + if posargs: + newcommand += " ".join(posargs) + continue + elif word.startswith("{posargs:") and word.endswith("}"): + if posargs: + newcommand += " ".join(posargs) + continue + else: + word = word[9:-1] + new_arg = "" + new_word = reader._replace(word) + new_word = reader._replace(new_word) + new_arg += new_word + newcommand += new_arg + + # Construct shlex object that will not escape any values, + # use all values as is in argv. + shlexer = shlex.shlex(newcommand, posix=True) + shlexer.whitespace_split = True + shlexer.escape = '' + shlexer.commenters = '' + argv = list(shlexer) + return argv class CommandParser(object): diff --git a/tox/_venv.py b/tox/_venv.py index 58587e4..a6c6de5 100644 --- a/tox/_venv.py +++ b/tox/_venv.py @@ -10,17 +10,17 @@ from tox._config import DepConfig class CreationConfig: def __init__(self, md5, python, version, sitepackages, - develop, deps): + usedevelop, deps): self.md5 = md5 self.python = python self.version = version self.sitepackages = sitepackages - self.develop = develop + self.usedevelop = usedevelop self.deps = deps def writeconfig(self, path): lines = ["%s %s" % (self.md5, self.python)] - lines.append("%s %d %d" % (self.version, self.sitepackages, self.develop)) + lines.append("%s %d %d" % (self.version, self.sitepackages, self.usedevelop)) for dep in self.deps: lines.append("%s %s" % dep) path.ensure() @@ -32,14 +32,14 @@ class CreationConfig: lines = path.readlines(cr=0) value = lines.pop(0).split(None, 1) md5, python = value - version, sitepackages, develop = lines.pop(0).split(None, 3) + version, sitepackages, usedevelop = lines.pop(0).split(None, 3) sitepackages = bool(int(sitepackages)) - develop = bool(int(develop)) + usedevelop = bool(int(usedevelop)) deps = [] for line in lines: md5, depstring = line.split(None, 1) deps.append((md5, depstring)) - return CreationConfig(md5, python, version, sitepackages, develop, deps) + return CreationConfig(md5, python, version, sitepackages, usedevelop, deps) except Exception: return None @@ -48,7 +48,7 @@ class CreationConfig: and self.python == other.python and self.version == other.version and self.sitepackages == other.sitepackages - and self.develop == other.develop + and self.usedevelop == other.usedevelop and self.deps == other.deps) @@ -147,7 +147,7 @@ class VirtualEnv(object): md5 = getdigest(python) version = tox.__version__ sitepackages = self.envconfig.sitepackages - develop = self.envconfig.develop + develop = self.envconfig.usedevelop deps = [] for dep in self._getresolvedeps(): raw_dep = dep.name @@ -321,11 +321,13 @@ class VirtualEnv(object): for envname in self.envconfig.passenv: if envname in os.environ: env[envname] = os.environ[envname] - setenv = self.envconfig.setenv - if setenv: - env.update(setenv) + + env.update(self.envconfig.setenv) + env['VIRTUAL_ENV'] = str(self.path) + env.update(extraenv) + return env def test(self, redirect=False): |
