Make it possible to have commands that contain (amongst other things) regex character classes

4 files changed, 217 insertions, 18 deletions

diff --git a/setup.py b/setup.py
index 349a8a9..dbfb8cd 100644
--- a/setup.py
+++ b/setup.py
@@ -59,4 +59,4 @@ def main():
     )
 
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()
diff --git a/tests/test_config.py b/tests/test_config.py
index 3f9c93d..3ad22ef 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1,8 +1,9 @@
-
 import tox
 import os, sys
+from textwrap import dedent
+
 import py
-from tox._config import IniReader
+from tox._config import IniReader, CommandParser
 
 class TestVenvConfig:
     def test_config_parsing_minimal(self, tmpdir, newconfig):
@@ -187,8 +188,8 @@ class TestIniParser:
             [section]
             key2=
                 cmd1 []
-                cmd2 [{item2} \
-                     other]
+                cmd2 {posargs:{item2} \
+                     other}
         """)
         reader = IniReader(config._cfg)
         posargs = ['hello', 'world']
@@ -209,6 +210,25 @@ class TestIniParser:
         assert argvlist[0] == ["cmd1"]
         assert argvlist[1] == ["cmd2", "value2", "other"]
 
+    def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig):
+        config = newconfig("""
+            [section]
+            key=
+                cmd0 []
+                cmd1 -m '[abc]'
+                cmd2 -m '\'something\'' []
+                cmd3 something[]else
+        """)
+        reader = IniReader(config._cfg)
+        posargs = ['hello', 'world']
+        reader.addsubstitions(posargs)
+
+        argvlist = reader.getargvlist('section', 'key')
+        assert argvlist[0] == ['cmd0'] + posargs
+        assert argvlist[1] == ['cmd1', '-m', '[abc]']
+        assert argvlist[2] == ['cmd2', '-m', "something"] + posargs
+        assert argvlist[3] == ['cmd3', 'something[]else']
+
     def test_getpath(self, tmpdir, newconfig):
         config = newconfig("""
             [section]
@@ -352,20 +372,24 @@ class TestConfigTestEnv:
             commands =
                 cmd1 [hello] \
                      world
+                cmd1 {posargs:hello} \
+                     world
         """
         conf = newconfig([], inisource).envconfigs['py24']
         argv = conf.commands
-        assert argv[0] == ["cmd1", "hello", "world"]
+        assert argv[0] == ["cmd1", "[hello]", "world"]
+        assert argv[1] == ["cmd1", "hello", "world"]
         conf = newconfig(['brave', 'new'], inisource).envconfigs['py24']
         argv = conf.commands
-        assert argv[0] == ["cmd1", "brave", "new", "world"]
+        assert argv[0] == ["cmd1", "[hello]", "world"]
+        assert argv[1] == ["cmd1", "brave", "new", "world"]
 
     def test_rewrite_posargs(self, tmpdir, newconfig):
         inisource = """
             [testenv:py24]
             args_are_paths = True
             changedir = tests
-            commands = cmd1 [hello]
+            commands = cmd1 {posargs:hello}
         """
         conf = newconfig([], inisource).envconfigs['py24']
         argv = conf.commands
@@ -582,3 +606,47 @@ class TestCmdInvocation:
             "*ERROR*tox.ini*does not exist*",
         ])
 
+
+class TestCommandParser:
+
+    def test_command_parser_for_word(self):
+        p = CommandParser('word')
+        assert list(p.words()) == ['word']
+
+    def test_command_parser_for_posargs(self):
+        p = CommandParser('[]')
+        assert list(p.words()) == ['[]']
+
+    def test_command_parser_for_multiple_words(self):
+        p = CommandParser('w1 w2 w3 ')
+        assert list(p.words()) == ['w1', 'w2', 'w3']
+
+    def test_command_parser_for_substitution_with_spaces(self):
+        p = CommandParser('{sub:something with spaces}')
+        assert list(p.words()) == ['{sub:something with spaces}']
+
+    def test_command_parser_with_complex_word_set(self):
+        complex_case = 'word [] [literal] {something} {some:other thing} w{ord} w{or}d w{ord} w{o:rd} w{o:r}d {w:or}d w[]ord {posargs:{a key}}'
+        p = CommandParser(complex_case)
+        parsed = list(p.words())
+        expected = [
+            'word', '[]', '[literal]', '{something}', '{some:other thing}',
+            'w{ord}', 'w{or}d', 'w{ord}', 'w{o:rd}', 'w{o:r}d', '{w:or}d',
+            'w[]ord', '{posargs:{a key}}',
+            ]
+
+        assert parsed == expected
+
+    def test_command_with_runs_of_whitespace(self):
+        cmd = dedent("""cmd1 {item1}
+                     {item2}""")
+        p = CommandParser(cmd)
+        parsed = list(p.words())
+        assert parsed == ['cmd1', '{item1}', '{item2}']
+
+    def test_command_with_split_line_in_subst_arguments(self):
+        cmd = dedent(""" cmd2 {posargs:{item2}
+                         other}""")
+        p = CommandParser(cmd)
+        parsed = list(p.words())
+        assert parsed == ['cmd2', '{posargs:{item2} other}']
diff --git a/tox.ini b/tox.ini
index c9322d6..6984ae1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,6 @@
 [tox]
 envlist=py27,py26,py25,py24,py31,docs
+#envlist=py27
 indexserver =
     testrun = http://pypi.testrun.org
     pypi = http://pypi.python.org/simple
diff --git a/tox/_config.py b/tox/_config.py
index 3991ad5..a2a891e 100644
--- a/tox/_config.py
+++ b/tox/_config.py
@@ -1,8 +1,13 @@
-import os, sys
-import py
+import argparse
+import os
+import sys
 import re
+import shlex
+import string
+
+import py
+
 import tox
-import argparse
 
 defaultenvs = {'jython': 'jython', 'pypy': 'pypy'}
 for _name in "py24,py25,py26,py27,py30,py31,py32".split(","):
@@ -144,7 +149,7 @@ class parseini:
                     if not url:
                         url = None
                 config.indexserver[name].url = url
-            
+
         reader.addsubstitions(toxworkdir=config.toxworkdir)
         config.distdir = reader.getpath(toxsection, "distdir",
                                            "{toxworkdir}/dist")
@@ -312,12 +317,25 @@ class IniReader:
 
     def _processcommand(self, command):
         posargs = self._subs.get('_posargs', None)
-        if posargs:
-            posargstring = " ".join(posargs)
-            command = re.sub("\[.*\]", lambda m: posargstring, command)
-        else:
-            command = command.replace("[", "").replace("]", "")
-        argv = [self._replace(x) for x in command.split()]
+
+        expression = r'\{(?:(?P<sub_type>[^:]+):)?(?P<substitution_value>.*)\}'
+
+        words = list(CommandParser(command).words())
+
+        new_command = ''
+        for word in words:
+            if word == '[]':
+                if posargs:
+                    new_command += ' '.join(posargs)
+                continue
+
+            new_word = re.sub(expression, self._replace_match, word)
+            # two passes; we might have substitutions in the result
+            new_word = re.sub(expression, self._replace_match, new_word)
+            new_command += new_word
+            new_command += ' '
+
+        argv = shlex.split(new_command.strip())
         return argv
 
     def getbool(self, section, name, default=None):
@@ -367,11 +385,123 @@ class IniReader:
                 "substitution key %r not found" % key)
         return str(self._subs[key])
 
+    def _replace_posargs(self, match):
+        posargs = self._subs.get('_posargs', None)
+
+        if posargs:
+            return " ".join(posargs)
+
+        if match.group('substitution_value'):
+            return match.group('substitution_value')
+        else:
+            return ''
+
+    def _replace_env(self, match):
+        envkey = match.group('substitution_value')
+        if not envkey:
+            raise tox.exception.ConfigError('env: requires an environment variable name')
+
+        if not envkey in os.environ:
+            raise tox.exception.ConfigError(
+                "substitution env:%r: %r not found in environment" %
+                (envkey, envkey))
+
+        return os.environ[envkey]
+
+    def _replace_substitution(self, match):
+        sub_key = match.group('substitution_value')
+        if sub_key not in self._subs:
+            raise tox.exception.ConfigError(
+                "substitution key %r not found" % sub_key)
+        return '"%s"' % str(self._subs[sub_key]).replace('"', r'\"')
+
+    def _replace_match(self, match):
+        g = match.groupdict()
+        handlers = {
+            'posargs' : self._replace_posargs,
+            'env' : self._replace_env,
+            None : self._replace_substitution,
+            }
+        try:
+            handler = handlers.get(g['sub_type'])
+        except KeyError:
+            raise tox.exception.ConfigError("No support for the %s substitution type" % g['sub_type'])
+        else:
+            return handler(match)
+
     def _replace(self, x, rexpattern = re.compile("\{.+?\}")):
         if '{' in x:
             return rexpattern.sub(self._sub, x)
         return x
 
+    def _parse_command(self, command):
+        pass
+
+class CommandParser(object):
+
+    def __init__(self, command):
+        self.command = command
+
+    @property
+    def cur_char(self):
+        return self.command[self.index]
+
+    def words(self):
+        self.index = 0
+        self.word = ''
+        self.can_yield = False
+        self.depth = 0
+        self.state = None
+        while self.index < len(self.command):
+            if self.cur_char in string.whitespace:
+                self.whitespace()
+
+            elif self.cur_char == '{':
+                self.can_yield = False
+                self.maybe_start_substitution()
+            elif self.cur_char == '}':
+                self.can_yield = False
+                self.maybe_end_substitution()
+            else:
+                self.can_yield = False
+                self.word += self.cur_char
+
+            self.index += 1
+
+            if self.can_yield and self.word.strip():
+                yield self.word
+                self.can_yield = False
+                self.word = ''
+
+        if self.word:
+            yield self.word
+
+    def whitespace(self):
+        if self.state == 'substitution':
+            if self.word and self.word[-1] not in string.whitespace:
+                self.word += ' '
+            return
+
+        self.can_yield = True
+
+    def maybe_start_substitution(self):
+        if self.state == 'substitution':
+            self.depth += 1
+        else:
+            assert self.depth == 0
+            self.state = 'substitution'
+
+        self.word += self.cur_char
+
+    def maybe_end_substitution(self):
+        if self.state == 'substitution':
+            if self.depth > 0:
+                self.depth -= 1
+            else:
+                self.state = None
+                assert self.depth == 0
+
+        self.word += self.cur_char
 
 def getcontextname():
     if 'HUDSON_URL' in os.environ: