Fix unescaped } breaking expansions that follow it (#1961)

Co-authored-by: Bernát Gábor <gaborjbernat@gmail.com>

5 files changed, 58 insertions, 42 deletions

diff --git a/docs/changelog/1956.bugfix.rst b/docs/changelog/1956.bugfix.rst
new file mode 100644
index 00000000..bd86986b
--- /dev/null
+++ b/docs/changelog/1956.bugfix.rst
@@ -0,0 +1,3 @@
+Due to a bug ``\{posargs} {posargs}`` used to expand to literal ``{posargs} {posargs}``.
+Now the second ``{posargs}`` is expanded.
+``\{posargs} {posargs}`` expands to ``{posargs} positional arguments here`` - by :user:`hexagonrecursion`.
diff --git a/src/tox/config/loader/ini/replace.py b/src/tox/config/loader/ini/replace.py
index 505519ce..c5074575 100644
--- a/src/tox/config/loader/ini/replace.py
+++ b/src/tox/config/loader/ini/replace.py
@@ -24,9 +24,9 @@ ARGS_GROUP = re.compile(r"(?<!\\\\|:[A-Z]):")
 
 def replace(conf: "Config", name: Optional[str], loader: "IniLoader", value: str, chain: List[str]) -> str:
     # perform all non-escaped replaces
-    start, end = 0, 0
+    end = 0
     while True:
-        start, end, to_replace = find_replace_part(value, start, end)
+        start, end, to_replace = find_replace_part(value, end)
         if to_replace is None:
             break
         replaced = _replace_match(conf, name, loader, to_replace, chain.copy())
@@ -34,10 +34,10 @@ def replace(conf: "Config", name: Optional[str], loader: "IniLoader", value: str
             # if we cannot replace, keep what was there, and continue looking for additional replaces following
             # note, here we cannot raise because the content may be a factorial expression, and in those case we don't
             # want to enforce escaping curly braces, e.g. it should work to write: env_list = {py39,py38}-{,dep}
-            start = end = end + 1
+            end = end + 1
             continue
         new_value = value[:start] + replaced + value[end + 1 :]
-        start, end = 0, 0  # if we performed a replacement start over
+        end = 0  # if we performed a replacement start over
         if new_value == value:  # if we're not making progress stop (circular reference?)
             break
         value = new_value
@@ -49,45 +49,25 @@ def replace(conf: "Config", name: Optional[str], loader: "IniLoader", value: str
     return value
 
 
-def find_replace_part(value: str, start: int, end: int) -> Tuple[int, int, Optional[str]]:
-    bracket_at = find_brackets(value, end)
-    if bracket_at != -1:
-        return bracket_at, bracket_at + 1, "posargs"  # brackets is an alias for positional arguments
-    start, end, match = find_braces(value, start, end)
-    return start, end, (value[start + 1 : end] if match else None)
-
-
-def find_brackets(value: str, end: int) -> int:
-    while True:
-        pos = value.find("[]", end)
-        if pos == -1:
-            break
-        if pos >= 1 and value[pos - 1] == "\\":  # the opened bracket is escaped
-            end = pos + 1
-            continue
-        break
-    return pos
+REPLACE_PART = re.compile(
+    r"""
+        (?<! \\) \{  # Unescaped {
+            ( [^{}] | \\ \{ | \\ \} )*  # Anything except an unescaped { or }
+        (?<! \\) \}  # Unescaped }
+    |
+        (?<! \\) \[ \]  # Unescaped []
+    """,
+    re.VERBOSE,
+)  # simplified - not verbose version (?<!\\)([^{}]|\\\{|\\\})*(?<!\\)\}|(?<!\\)\[\]
 
 
-def find_braces(value: str, start: int, end: int) -> Tuple[int, int, bool]:
-    match = False
-    while end != -1:
-        end = value.find("}", end)
-        if end == -1:
-            continue
-        if end >= 1 and value[end - 1] == "\\":  # ignore escaped
-            end += 1
-            continue
-        before = end
-        while True:
-            start = value.rfind("{", 0, before)
-            if start >= 1 and value[start - 1] == "\\":  # ignore escaped
-                before = start - 1
-                continue
-            match = start != -1
-            break
-        break  # pragma: no cover # for some odd reason this line is not reported by coverage, though is needed
-    return start, end, match
+def find_replace_part(value: str, end: int) -> Tuple[int, int, Optional[str]]:
+    match = REPLACE_PART.search(value, end)
+    if match is None:
+        return -1, -1, None
+    if match.group() == "[]":
+        return match.start(), match.end() - 1, "posargs"  # brackets is an alias for positional arguments
+    return match.start(), match.end() - 1, match.group()[1:-1]
 
 
 def _replace_match(
diff --git a/src/tox/config/set_env.py b/src/tox/config/set_env.py
index 8e5b5a49..8a4d794a 100644
--- a/src/tox/config/set_env.py
+++ b/src/tox/config/set_env.py
@@ -18,7 +18,7 @@ class SetEnv:
                     if "{" in key:
                         raise ValueError(f"invalid line {line!r} in set_env")
                 except ValueError:
-                    _, __, match = find_replace_part(line, 0, 0)
+                    _, __, match = find_replace_part(line, 0)
                     if match:
                         self._later.append(line)
                     else:
diff --git a/tests/config/loader/ini/replace/test_replace.py b/tests/config/loader/ini/replace/test_replace.py
new file mode 100644
index 00000000..99de5bcd
--- /dev/null
+++ b/tests/config/loader/ini/replace/test_replace.py
@@ -0,0 +1,27 @@
+from typing import Tuple
+
+import pytest
+
+from tox.config.loader.ini.replace import find_replace_part
+
+
+@pytest.mark.parametrize(
+    ("value", "result"),
+    [
+        ("[]", (0, 1, "posargs")),
+        ("123[]", (3, 4, "posargs")),
+        ("[]123", (0, 1, "posargs")),
+        (r"\[\] []", (5, 6, "posargs")),
+        (r"[\] []", (4, 5, "posargs")),
+        (r"\[] []", (4, 5, "posargs")),
+        ("{foo}", (0, 4, "foo")),
+        (r"\{foo} {bar}", (7, 11, "bar")),
+        ("{foo} {bar}", (0, 4, "foo")),
+        (r"{foo\} {bar}", (7, 11, "bar")),
+        (r"{foo:{bar}}", (5, 9, "bar")),
+        (r"{\{}", (0, 3, r"\{")),
+        (r"{\}}", (0, 3, r"\}")),
+    ],
+)
+def test_match(value: str, result: Tuple[int, int, str]) -> None:
+    assert find_replace_part(value, 0) == result
diff --git a/tests/config/loader/ini/replace/test_replace_posargs.py b/tests/config/loader/ini/replace/test_replace_posargs.py
index d2ebed4b..45b26ff8 100644
--- a/tests/config/loader/ini/replace/test_replace_posargs.py
+++ b/tests/config/loader/ini/replace/test_replace_posargs.py
@@ -75,3 +75,9 @@ def test_replace_pos_args_escaped(replace_one: ReplaceOne, value: str) -> None:
 def test_replace_mixed_brackets_and_braces(replace_one: ReplaceOne, value: str, result: str) -> None:
     outcome = replace_one(value, ["foo"])
     assert result == outcome
+
+
+def test_half_escaped_braces(replace_one: ReplaceOne) -> None:
+    """See https://github.com/tox-dev/tox/issues/1956"""
+    outcome = replace_one(r"\{posargs} {posargs}", ["foo"])
+    assert "{posargs} foo" == outcome