fix issue #120: section subs in commands doesn't work

Problem
-----------

Section substitution for `commands` doesn't work correctly.

Acceptance
----------------

When section substitution is specified as a single form of `commands` declaration
it is replaced with parsed list of commands.

When section substation happens as part of the other command line declaration - preserve original behaviour.

Changes
------------

- fixes for the issue
- some PEP8 violations are fixed
- tests for the substation in `commands` are grouped

2 files changed, 108 insertions, 33 deletions

diff --git a/tests/test_config.py b/tests/test_config.py
index f879918..b474c96 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -208,6 +208,52 @@ class TestGetcontextname:
         assert getcontextname() == "jenkins"
 
 
+class TestIniParserAgainstCommandsKey:
+    """Test parsing commands with substitutions"""
+
+    def test_command_substitution_from_other_section(self, newconfig):
+        config = newconfig("""
+            [section]
+            key = whatever
+            [testenv]
+            commands =
+                echo {[section]key}
+            """)
+        reader = IniReader(config._cfg)
+        x = reader.getargvlist("testenv", "commands")
+        assert x == [["echo", "whatever"]]
+
+    def test_command_substitution_from_other_section_multiline(self, newconfig):
+        """Ensure referenced multiline commands form from other section injected as multiple commands."""
+        config = newconfig("""
+            [section]
+            commands =
+                      cmd1 param11 param12
+                      # comment is omitted
+                      cmd2 param21 \
+                           param22
+            [base]
+            commands = cmd 1 \
+                           2 3 4
+                       cmd 2
+            [testenv]
+            commands =
+                {[section]commands}
+                {[section]commands}
+                # comment is omitted
+                echo {[base]commands}
+            """)
+        reader = IniReader(config._cfg)
+        x = reader.getargvlist("testenv", "commands")
+        assert x == [
+            "cmd1 param11 param12".split(),
+            "cmd2 param21 param22".split(),
+            "cmd1 param11 param12".split(),
+            "cmd2 param21 param22".split(),
+            ["echo", "cmd", "1", "2", "3", "4", "cmd", "2"],
+        ]
+
+
 class TestIniParser:
     def test_getdefault_single(self, tmpdir, newconfig):
         config = newconfig("""
@@ -318,6 +364,14 @@ class TestIniParser:
         x = reader.getdefault("section", "key3")
         assert x == ""
 
+    def test_value_matches_section_substituion(self):
+        assert is_section_substitution("{[setup]commands}")
+
+    def test_value_doesn_match_section_substitution(self):
+        assert is_section_substitution("{[ ]commands}") is None
+        assert is_section_substitution("{[setup]}") is None
+        assert is_section_substitution("{[setup] commands}") is None
+
     def test_getdefault_other_section_substitution(self, newconfig):
         config = newconfig("""
             [section]
@@ -329,18 +383,6 @@ class TestIniParser:
         x = reader.getdefault("testenv", "key")
         assert x == "true"
 
-    def test_command_substitution_from_other_section(self, newconfig):
-        config = newconfig("""
-            [section]
-            key = whatever
-            [testenv]
-            commands =
-                echo {[section]key}
-            """)
-        reader = IniReader(config._cfg)
-        x = reader.getargvlist("testenv", "commands")
-        assert x == [["echo", "whatever"]]
-
     def test_argvlist(self, tmpdir, newconfig):
         config = newconfig("""
             [section]
@@ -382,7 +424,6 @@ class TestIniParser:
         x = reader.getargvlist("section", "key2")
         assert x == [["cmd1", "with", "space", "grr"]]
 
-
     def test_argvlist_quoting_in_command(self, tmpdir, newconfig):
         config = newconfig("""
             [section]
@@ -394,7 +435,6 @@ class TestIniParser:
         x = reader.getargvlist("section", "key1")
         assert x == [["cmd1", "with space", "after the comment"]]
 
-
     def test_argvlist_positional_substitution(self, tmpdir, newconfig):
         config = newconfig("""
             [section]
@@ -483,7 +523,6 @@ class TestIniParser:
         expected = ['py.test', '-n5', '--junitxml=ENV_LOG_DIR/junit-ENV_NAME.xml', 'hello', 'world']
         assert reader.getargvlist('section', 'key')[0] == expected
 
-
     def test_getargv(self, newconfig):
         config = newconfig("""
             [section]
@@ -493,7 +532,6 @@ class TestIniParser:
         expected = ['some', 'command', 'with quoting']
         assert reader.getargv('section', 'key') == expected
 
-
     def test_getpath(self, tmpdir, newconfig):
         config = newconfig("""
             [section]
@@ -514,13 +552,14 @@ class TestIniParser:
             key5=yes
         """)
         reader = IniReader(config._cfg)
-        assert reader.getbool("section", "key1") == True
-        assert reader.getbool("section", "key1a") == True
-        assert reader.getbool("section", "key2") == False
-        assert reader.getbool("section", "key2a") == False
+        assert reader.getbool("section", "key1") is True
+        assert reader.getbool("section", "key1a") is True
+        assert reader.getbool("section", "key2") is False
+        assert reader.getbool("section", "key2a") is False
         py.test.raises(KeyError, 'reader.getbool("section", "key3")')
         py.test.raises(tox.exception.ConfigError, 'reader.getbool("section", "key5")')
 
+
 class TestConfigTestEnv:
     def test_commentchars_issue33(self, tmpdir, newconfig):
         config = newconfig("""
@@ -1559,7 +1598,7 @@ class TestCommandParser:
             'word', ' ', '[]', ' ', '[literal]', ' ', '{something}', ' ', '{some:other thing}',
             ' ', 'w', '{ord}', ' ', 'w', '{or}', 'd', ' ', 'w', '{ord}', ' ', 'w', '{o:rd}', ' ', 'w', '{o:r}', 'd', ' ', '{w:or}', 'd',
             ' ', 'w[]ord', ' ', '{posargs:{a key}}',
-            ]
+        ]
 
         assert parsed == expected
 
@@ -1582,7 +1621,6 @@ class TestCommandParser:
         parsed = list(p.words())
         assert parsed == ['nosetests', ' ', '-v', ' ', '-a', ' ', '!deferred', ' ', '--with-doctest', ' ', '[]']
 
-
     @pytest.mark.skipif("sys.platform != 'win32'")
     def test_commands_with_backslash(self, newconfig):
         config = newconfig([r"hello\world"], """
diff --git a/tox/_config.py b/tox/_config.py
index 80d61bb..9d054a0 100644
--- a/tox/_config.py
+++ b/tox/_config.py
@@ -22,7 +22,14 @@ default_factors = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3',
 for version in '24,25,26,27,30,31,32,33,34,35'.split(','):
     default_factors['py' + version] = 'python%s.%s' % tuple(version)
 
+
 def parseconfig(args=None, pkg=None):
+    """
+    :param list[str] args: Optional list of arguments.
+    :type pkg: str
+    :rtype: :class:`Config`
+    :raise SystemExit: toxinit file is not found
+    """
     if args is None:
         args = sys.argv[1:]
     parser = prepare_parse(pkg)
@@ -509,16 +516,23 @@ class DepConfig:
     def __str__(self):
         if self.indexserver:
             if self.indexserver.name == "default":
-               return self.name
-            return ":%s:%s" %(self.indexserver.name, self.name)
+                return self.name
+            return ":%s:%s" % (self.indexserver.name, self.name)
         return str(self.name)
     __repr__ = __str__
 
+
 class IndexServerConfig:
     def __init__(self, name, url=None):
         self.name = name
         self.url = url
 
+
+#: Check value matches substitution form
+#: of referencing value from other section. E.g. {[base]commands}
+is_section_substitution = re.compile("{\[[^{}\s]+\]\S+?}").match
+
+
 RE_ITEM_REF = re.compile(
     r'''
     (?<!\\)[{]
@@ -570,13 +584,31 @@ class IniReader:
         return value
 
     def getargvlist(self, section, name):
-        s = self.getdefault(section, name, '', replace=False)
-        #if s is None:
-        #    raise tox.exception.ConfigError(
-        #        "no command list %r defined in section [%s]" %(name, section))
-        commandlist = []
+        """Get arguments for every parsed command.
+
+        :param str section: Section name in the configuration.
+        :param str name: Key name in a section.
+        :rtype: list[list[str]]
+        :raise :class:`tox.exception.ConfigError`:
+            line-continuation ends nowhere while resolving for specified section
+        """
+        content = self.getdefault(section, name, '', replace=False)
+        return self._parse_commands(section, name, content)
+
+    def _parse_commands(self, section, name, content):
+        """Parse commands from key content in specified section.
+
+        :param str section: Section name in the configuration.
+        :param str name: Key name in a section.
+        :param str content: Content stored by key.
+
+        :rtype: list[list[str]]
+        :raise :class:`tox.exception.ConfigError`:
+            line-continuation ends nowhere while resolving for specified section
+        """
+        commands = []
         current_command = ""
-        for line in s.split("\n"):
+        for line in content.splitlines():
             line = line.rstrip()
             i = line.find("#")
             if i != -1:
@@ -587,14 +619,19 @@ class IniReader:
                 current_command += " " + line[:-1]
                 continue
             current_command += line
-            commandlist.append(self._processcommand(current_command))
+
+            if is_section_substitution(current_command):
+                replaced = self._replace(current_command)
+                commands.extend(self._parse_commands(section, name, replaced))
+            else:
+                commands.append(self._processcommand(current_command))
             current_command = ""
         else:
             if current_command:
                 raise tox.exception.ConfigError(
-                    "line-continuation for [%s] %s ends nowhere" %
+                    "line-continuation ends nowhere while resolving for [%s] %s" %
                     (section, name))
-        return commandlist
+        return commands
 
     def _processcommand(self, command):
         posargs = getattr(self, "posargs", None)