Adds the walrus operator to the Python3 lexer.

- The walrus operator, also known as assignment expressions, was introduced in Python 3.8
- Moves the Token.Operator matching root above Token.Punctuation so the walrus operator takes precedence
- Includes a test to make sure this behavior doesn't regress since it's sensitive to the order of expressions
- Fixes #1381

1 files changed, 25 insertions, 0 deletions

diff --git a/tests/test_python.py b/tests/test_python.py
index 4e5d5bbf..b0922adf 100644
--- a/tests/test_python.py
+++ b/tests/test_python.py
@@ -133,3 +133,28 @@ def test_pep_515(lexer3):
             (Token.Text, u'\n'),
         ]
         assert list(lexer3.get_tokens(fragment)) == tokens
+
+
+def test_walrus_operator(lexer3):
+    """
+    Tests that ':=' is recognized as an Operator
+    """
+    fragment = u'if (a := 2) > 4:'
+    tokens = [
+        (Token.Keyword, 'if'),
+        (Token.Text, ' '),
+        (Token.Punctuation, '('),
+        (Token.Name, 'a'),
+        (Token.Text, ' '),
+        (Token.Operator, ':='),
+        (Token.Text, ' '),
+        (Token.Literal.Number.Integer, '2'),
+        (Token.Punctuation, ')'),
+        (Token.Text, ' '),
+        (Token.Operator, '>'),
+        (Token.Text, ' '),
+        (Token.Literal.Number.Integer, '4'),
+        (Token.Punctuation, ':'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer3.get_tokens(fragment)) == tokens