From 19d2096185de3282345eab1da611e56a26bcaec2 Mon Sep 17 00:00:00 2001
From: Jeppe Dakin <jeppe_dakin@hotmail.com>
Date: Sun, 17 Jan 2021 12:54:52 +0100
Subject: Fix for lexing Python raw f-strings with backslashes (#1683)

* introduce and apply rfstringescape

* add unit test for raw f-strings

* add further tests

* fix comment
---
 tests/test_python.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

(limited to 'tests/test_python.py')

diff --git a/tests/test_python.py b/tests/test_python.py
index 188044fa..ee36a331 100644
--- a/tests/test_python.py
+++ b/tests/test_python.py
@@ -840,3 +840,51 @@ def test_fstring(lexer3):
             match = pattern.sub(lambda m: rep[m.group(0)], match)
             tokens[i] = (token, match)
         assert list(lexer3.get_tokens(fragment)) == tokens
+
+def test_raw_fstring(lexer3):
+    """
+    Tests that the lexer can parse raw f-strings
+    """
+    # Just raw
+    fragment = r'rf"m_\nu = x"'
+    tokens = [
+        (Token.Literal.String.Affix, 'rf'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Literal.String.Double, 'm_'),
+        (Token.Literal.String.Double, '\\'),
+        (Token.Literal.String.Double, 'nu = x'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Text, '\n')
+    ]
+    # Just f-string
+    fragment = r'f"m_\nu = {x}"'
+    tokens = [
+        (Token.Literal.String.Affix, 'f'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Literal.String.Double, 'm_'),
+        (Token.Literal.String.Escape, '\\n'),
+        (Token.Literal.String.Double, 'u = '),
+        (Token.Literal.String.Interpol, '{'),
+        (Token.Name, 'x'),
+        (Token.Literal.String.Interpol, '}'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Text, '\n'),
+    ]
+    # Raw behavior inside {{...}}
+    fragment = r'rf"m_{{\nu}} = {x}"'
+    tokens = [
+        (Token.Literal.String.Affix, 'rf'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Literal.String.Double, 'm_'),
+        (Token.Literal.String.Escape, '{{'),
+        (Token.Literal.String.Double, '\\'),
+        (Token.Literal.String.Double, 'nu'),
+        (Token.Literal.String.Escape, '}}'),
+        (Token.Literal.String.Double, ' = '),
+        (Token.Literal.String.Interpol, '{'),
+        (Token.Name, 'x'),
+        (Token.Literal.String.Interpol, '}'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer3.get_tokens(fragment)) == tokens
-- 
cgit v1.2.1