Fix an error with parsing commands that have multiple substitutions

2 files changed, 93 insertions, 57 deletions

diff --git a/tests/test_config.py b/tests/test_config.py
index c5438c3..a878178 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -241,6 +241,20 @@ class TestIniParser:
         assert argvlist[2] == ['cmd2', '-m', "something"] + posargs
         assert argvlist[3] == ['cmd3', 'something[]else']
 
+    def test_substition_with_multiple_words(self, newconfig):
+        inisource = """
+            [section]
+            key = py.test -n5 --junitxml={envlogdir}/junit-{envname}.xml []
+            """
+        config = newconfig(inisource)
+        reader = IniReader(config._cfg)
+        posargs = ['hello', 'world']
+        reader.addsubstitions(posargs, envlogdir='ENV_LOG_DIR', envname='ENV_NAME')
+
+        expected = ['py.test', '-n5', '--junitxml=ENV_LOG_DIR/junit-ENV_NAME.xml', 'hello', 'world']
+        assert reader.getargvlist('section', 'key')[0] == expected
+
+
     def test_getpath(self, tmpdir, newconfig):
         config = newconfig("""
             [section]
@@ -623,6 +637,7 @@ class TestCommandParser:
 
     def test_command_parser_for_word(self):
         p = CommandParser('word')
+        # import pytest; pytest.set_trace()
         assert list(p.words()) == ['word']
 
     def test_command_parser_for_posargs(self):
@@ -631,7 +646,7 @@ class TestCommandParser:
 
     def test_command_parser_for_multiple_words(self):
         p = CommandParser('w1 w2 w3 ')
-        assert list(p.words()) == ['w1', 'w2', 'w3']
+        assert list(p.words()) == ['w1', ' ', 'w2', ' ', 'w3']
 
     def test_command_parser_for_substitution_with_spaces(self):
         p = CommandParser('{sub:something with spaces}')
@@ -642,9 +657,9 @@ class TestCommandParser:
         p = CommandParser(complex_case)
         parsed = list(p.words())
         expected = [
-            'word', '[]', '[literal]', '{something}', '{some:other thing}',
-            'w{ord}', 'w{or}d', 'w{ord}', 'w{o:rd}', 'w{o:r}d', '{w:or}d',
-            'w[]ord', '{posargs:{a key}}',
+            'word', ' ', '[]', ' ', '[literal]', ' ', '{something}', ' ', '{some:other thing}',
+            ' ', 'w', '{ord}', ' ', 'w', '{or}', 'd', ' ', 'w', '{ord}', ' ', 'w', '{o:rd}', ' ', 'w', '{o:r}', 'd', ' ', '{w:or}', 'd',
+            ' ', 'w[]ord', ' ', '{posargs:{a key}}',
             ]
 
         assert parsed == expected
@@ -654,17 +669,17 @@ class TestCommandParser:
                      {item2}""")
         p = CommandParser(cmd)
         parsed = list(p.words())
-        assert parsed == ['cmd1', '{item1}', '{item2}']
+        assert parsed == ['cmd1', ' ', '{item1}', ' ', '{item2}']
 
     def test_command_with_split_line_in_subst_arguments(self):
         cmd = dedent(""" cmd2 {posargs:{item2}
                          other}""")
         p = CommandParser(cmd)
         parsed = list(p.words())
-        assert parsed == ['cmd2', '{posargs:{item2} other}']
+        assert parsed == ['cmd2', ' ', '{posargs:{item2}\n                        other}']
 
     def test_command_parsing_for_issue_10(self):
         cmd = "nosetests -v -a !deferred --with-doctest []"
         p = CommandParser(cmd)
         parsed = list(p.words())
-        assert parsed == ['nosetests', '-v', '-a', '!deferred', '--with-doctest', '[]']
+        assert parsed == ['nosetests', ' ', '-v', ' ', '-a', ' ', '!deferred', ' ', '--with-doctest', ' ', '[]']
diff --git a/tox/_config.py b/tox/_config.py
index a2a891e..98419d9 100644
--- a/tox/_config.py
+++ b/tox/_config.py
@@ -333,7 +333,6 @@ class IniReader:
             # two passes; we might have substitutions in the result
             new_word = re.sub(expression, self._replace_match, new_word)
             new_command += new_word
-            new_command += ' '
 
         argv = shlex.split(new_command.strip())
         return argv
@@ -439,69 +438,91 @@ class IniReader:
 
 class CommandParser(object):
 
+    class State(object):
+        def __init__(self):
+            self.index = 0
+            self.word = ''
+            self.depth = 0
+            self.yield_word = None
+            self.state = 'before_start'
+
     def __init__(self, command):
         self.command = command
 
-    @property
-    def cur_char(self):
-        return self.command[self.index]
-
     def words(self):
-        self.index = 0
-        self.word = ''
-        self.can_yield = False
-        self.depth = 0
-        self.state = None
-        while self.index < len(self.command):
-            if self.cur_char in string.whitespace:
-                self.whitespace()
-
-            elif self.cur_char == '{':
-                self.can_yield = False
-                self.maybe_start_substitution()
-            elif self.cur_char == '}':
-                self.can_yield = False
-                self.maybe_end_substitution()
-            else:
-                self.can_yield = False
-                self.word += self.cur_char
+        ps = CommandParser.State()
 
-            self.index += 1
+        def cur_char():
+            return self.command[ps.index]
 
-            if self.can_yield and self.word.strip():
-                yield self.word
-                self.can_yield = False
-                self.word = ''
+        def word_has_ended():
+            return ((cur_char() in string.whitespace and ps.word and ps.word[-1] not in string.whitespace) or
+                    (cur_char() == '{' and not ps.state == 'substitution') or
+                    (ps.state is not 'substitution' and ps.word and ps.word[-1] == '}') or
+                    (cur_char() not in string.whitespace and ps.word and ps.word.strip() == ''))
+            return (ps.state is None and
+                    (ps.word.endswith('}') or
+                     ps.word.strip() == ''))
 
-        if self.word:
-            yield self.word
+        def yield_this_word():
+            ps.yield_word = ps.word
+            ps.word = ''
 
-    def whitespace(self):
-        if self.state == 'substitution':
-            if self.word and self.word[-1] not in string.whitespace:
-                self.word += ' '
-            return
+        def accumulate():
+            ps.word += cur_char()
 
-        self.can_yield = True
+        def push_substitution():
+            if ps.depth == 0:
+                ps.state = 'substitution'
+            ps.depth += 1
 
-    def maybe_start_substitution(self):
-        if self.state == 'substitution':
-            self.depth += 1
-        else:
-            assert self.depth == 0
-            self.state = 'substitution'
+        def pop_substitution():
+            ps.depth -= 1
+            if ps.depth == 0:
+                ps.state = None
+
+        while ps.index < len(self.command):
+
+            if cur_char() in string.whitespace:
+                if ps.state == 'substitution':
+                    accumulate()
+
+                else:
+                    if word_has_ended():
+                        yield_this_word()
 
-        self.word += self.cur_char
+                    accumulate()
+
+            elif cur_char() == '{':
+                if word_has_ended():
+                    yield_this_word()
+
+                accumulate()
+                push_substitution()
+
+            elif cur_char() == '}':
+                accumulate()
+                pop_substitution()
 
-    def maybe_end_substitution(self):
-        if self.state == 'substitution':
-            if self.depth > 0:
-                self.depth -= 1
             else:
-                self.state = None
-                assert self.depth == 0
+                if word_has_ended():
+                    yield_this_word()
+
+                accumulate()
+
+            ps.index += 1
+
+            if ps.yield_word:
+                if ps.yield_word.strip():
+                    yield ps.yield_word
+                else:
+                    yield ' '
+
+                ps.yield_word = None
+
+        if ps.word.strip():
+            yield ps.word.strip()
 
-        self.word += self.cur_char
 
 def getcontextname():
     if 'HUDSON_URL' in os.environ: