Support error locations in Python tracebacks (#1852)

Support both single carets for syntax errors (Python 2 and 3)
and fine-grained error locations with several carets (Python 3.11+).
Previously, the carets were highlighted as operators.  This uses
a new token, Token.Punctuation.Marker.  For now, no style supports
it specifically.  In the future, styles might start differentiating
it from Token.Punctuation.

[Closes #1850.]

13 files changed, 214 insertions, 7 deletions

diff --git a/tests/examplefiles/pycon/pycon_ctrlc_traceback.output b/tests/examplefiles/pycon/pycon_ctrlc_traceback.output
index e977d7ae..2cdae0cf 100644
--- a/tests/examplefiles/pycon/pycon_ctrlc_traceback.output
+++ b/tests/examplefiles/pycon/pycon_ctrlc_traceback.output
@@ -271,9 +271,8 @@
 '='           Operator
 '\n'          Text
 
-'    '        Text
-'   '         Text
-'^'           Operator
+'       '     Text
+'^'           Punctuation.Marker
 '\n'          Text
 
 'SyntaxError' Generic.Error
diff --git a/tests/examplefiles/pycon/pycon_test.pycon.output b/tests/examplefiles/pycon/pycon_test.pycon.output
index abeb6ec1..2ab6654b 100644
--- a/tests/examplefiles/pycon/pycon_test.pycon.output
+++ b/tests/examplefiles/pycon/pycon_test.pycon.output
@@ -14,7 +14,7 @@
 '\n'          Text
 
 '    '        Text
-'^'           Operator
+'^'           Punctuation.Marker
 '\n'          Text
 
 'SyntaxError' Generic.Error
diff --git a/tests/examplefiles/pytb/error_locations.pytb b/tests/examplefiles/pytb/error_locations.pytb
new file mode 100644
index 00000000..b82fb90b
--- /dev/null
+++ b/tests/examplefiles/pytb/error_locations.pytb
@@ -0,0 +1,17 @@
+Traceback (most recent call last):
+  File "/home/tb.py", line 13, in <module>
+    in_Python_3_11()
+    ^^^^^^^^^^^^^^^^
+  File "/home/tb.py", line 2, in in_Python_3_11
+    return error_locations()
+           ^^^^^^^^^^^^^^^^^
+  File "/home/tb.py", line 5, in error_locations
+    return are_provided() is True
+           ^^^^^^^^^^^^^^
+  File "/home/tb.py", line 8, in are_provided
+    return in_tracebacks()
+           ^^^^^^^^^^^^^^^
+  File "/home/tb.py", line 11, in in_tracebacks
+    return 1/0
+           ^^^
+ZeroDivisionError: division by zero
diff --git a/tests/examplefiles/pytb/error_locations.pytb.output b/tests/examplefiles/pytb/error_locations.pytb.output
new file mode 100644
index 00000000..361aa3fb
--- /dev/null
+++ b/tests/examplefiles/pytb/error_locations.pytb.output
@@ -0,0 +1,108 @@
+'Traceback (most recent call last):\n' Generic.Traceback
+
+'  File '     Text
+'"/home/tb.py"' Name.Builtin
+', line '     Text
+'13'          Literal.Number
+', in '       Text
+'<module>'    Name
+'\n'          Text
+
+'    '        Text
+'in_Python_3_11' Name
+'('           Punctuation
+')'           Punctuation
+'\n'          Text
+
+'    '        Text
+'^^^^^^^^^^^^^^^^' Punctuation.Marker
+'\n'          Text
+
+'  File '     Text
+'"/home/tb.py"' Name.Builtin
+', line '     Text
+'2'           Literal.Number
+', in '       Text
+'in_Python_3_11' Name
+'\n'          Text
+
+'    '        Text
+'return'      Keyword
+' '           Text
+'error_locations' Name
+'('           Punctuation
+')'           Punctuation
+'\n'          Text
+
+'           ' Text
+'^^^^^^^^^^^^^^^^^' Punctuation.Marker
+'\n'          Text
+
+'  File '     Text
+'"/home/tb.py"' Name.Builtin
+', line '     Text
+'5'           Literal.Number
+', in '       Text
+'error_locations' Name
+'\n'          Text
+
+'    '        Text
+'return'      Keyword
+' '           Text
+'are_provided' Name
+'('           Punctuation
+')'           Punctuation
+' '           Text
+'is'          Operator.Word
+' '           Text
+'True'        Keyword.Constant
+'\n'          Text
+
+'           ' Text
+'^^^^^^^^^^^^^^' Punctuation.Marker
+'\n'          Text
+
+'  File '     Text
+'"/home/tb.py"' Name.Builtin
+', line '     Text
+'8'           Literal.Number
+', in '       Text
+'are_provided' Name
+'\n'          Text
+
+'    '        Text
+'return'      Keyword
+' '           Text
+'in_tracebacks' Name
+'('           Punctuation
+')'           Punctuation
+'\n'          Text
+
+'           ' Text
+'^^^^^^^^^^^^^^^' Punctuation.Marker
+'\n'          Text
+
+'  File '     Text
+'"/home/tb.py"' Name.Builtin
+', line '     Text
+'11'          Literal.Number
+', in '       Text
+'in_tracebacks' Name
+'\n'          Text
+
+'    '        Text
+'return'      Keyword
+' '           Text
+'1'           Literal.Number.Integer
+'/'           Operator
+'0'           Literal.Number.Integer
+'\n'          Text
+
+'           ' Text
+'^^^'         Punctuation.Marker
+'\n'          Text
+
+'ZeroDivisionError' Generic.Error
+': '          Text
+'division by zero' Name
+'\n'          Text
diff --git a/tests/examplefiles/pytb/py3tb_test.py3tb.output b/tests/examplefiles/pytb/py3tb_test.py3tb.output
index 9b760224..05445c72 100644
--- a/tests/examplefiles/pytb/py3tb_test.py3tb.output
+++ b/tests/examplefiles/pytb/py3tb_test.py3tb.output
@@ -10,9 +10,8 @@
 '+'           Operator
 '\n'          Text
 
-'    '        Text
-' '           Text
-'^'           Operator
+'     '       Text
+'^'           Punctuation.Marker
 '\n'          Text
 
 'SyntaxError' Generic.Error
diff --git a/tests/examplefiles/pytb/syntax_error.py2tb b/tests/examplefiles/pytb/syntax_error.py2tb
new file mode 100644
index 00000000..3525cee2
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error.py2tb
@@ -0,0 +1,4 @@
+  File "<stdin>", line 1
+    mismatch(]
+           ^
+SyntaxError: closing parenthesis ']' does not match opening parenthesis '('
diff --git a/tests/examplefiles/pytb/syntax_error.py2tb.output b/tests/examplefiles/pytb/syntax_error.py2tb.output
new file mode 100644
index 00000000..f7ae91bf
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error.py2tb.output
@@ -0,0 +1,21 @@
+''            Generic.Traceback
+'  File '     Text
+'"<stdin>"'   Name.Builtin
+', line '     Text
+'1'           Literal.Number
+'\n'          Text
+
+'    '        Text
+'mismatch'    Name
+'('           Punctuation
+']'           Punctuation
+'\n'          Text
+
+'           ' Text
+'^'           Punctuation.Marker
+'\n'          Text
+
+'SyntaxError' Generic.Error
+': '          Text
+"closing parenthesis ']' does not match opening parenthesis '('" Name
+'\n'          Text
diff --git a/tests/examplefiles/pytb/syntax_error.pytb b/tests/examplefiles/pytb/syntax_error.pytb
new file mode 100644
index 00000000..3525cee2
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error.pytb
@@ -0,0 +1,4 @@
+  File "<stdin>", line 1
+    mismatch(]
+           ^
+SyntaxError: closing parenthesis ']' does not match opening parenthesis '('
diff --git a/tests/examplefiles/pytb/syntax_error.pytb.output b/tests/examplefiles/pytb/syntax_error.pytb.output
new file mode 100644
index 00000000..f7ae91bf
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error.pytb.output
@@ -0,0 +1,21 @@
+''            Generic.Traceback
+'  File '     Text
+'"<stdin>"'   Name.Builtin
+', line '     Text
+'1'           Literal.Number
+'\n'          Text
+
+'    '        Text
+'mismatch'    Name
+'('           Punctuation
+']'           Punctuation
+'\n'          Text
+
+'           ' Text
+'^'           Punctuation.Marker
+'\n'          Text
+
+'SyntaxError' Generic.Error
+': '          Text
+"closing parenthesis ']' does not match opening parenthesis '('" Name
+'\n'          Text
diff --git a/tests/examplefiles/pytb/syntax_error_caret_code.py2tb b/tests/examplefiles/pytb/syntax_error_caret_code.py2tb
new file mode 100644
index 00000000..9a021002
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error_caret_code.py2tb
@@ -0,0 +1,3 @@
+  File "<stdin>", line 1
+    ^
+    ^
diff --git a/tests/examplefiles/pytb/syntax_error_caret_code.py2tb.output b/tests/examplefiles/pytb/syntax_error_caret_code.py2tb.output
new file mode 100644
index 00000000..8e51a868
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error_caret_code.py2tb.output
@@ -0,0 +1,14 @@
+''            Generic.Traceback
+'  File '     Text
+'"<stdin>"'   Name.Builtin
+', line '     Text
+'1'           Literal.Number
+'\n'          Text
+
+'    '        Text
+'^'           Operator
+'\n'          Text
+
+'    '        Text
+'^'           Punctuation.Marker
+'\n'          Text
diff --git a/tests/examplefiles/pytb/syntax_error_caret_code.pytb b/tests/examplefiles/pytb/syntax_error_caret_code.pytb
new file mode 100644
index 00000000..9a021002
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error_caret_code.pytb
@@ -0,0 +1,3 @@
+  File "<stdin>", line 1
+    ^
+    ^
diff --git a/tests/examplefiles/pytb/syntax_error_caret_code.pytb.output b/tests/examplefiles/pytb/syntax_error_caret_code.pytb.output
new file mode 100644
index 00000000..8e51a868
--- /dev/null
+++ b/tests/examplefiles/pytb/syntax_error_caret_code.pytb.output
@@ -0,0 +1,14 @@
+''            Generic.Traceback
+'  File '     Text
+'"<stdin>"'   Name.Builtin
+', line '     Text
+'1'           Literal.Number
+'\n'          Text
+
+'    '        Text
+'^'           Operator
+'\n'          Text
+
+'    '        Text
+'^'           Punctuation.Marker
+'\n'          Text