1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
|
---input---
1 + 1 (* This is a comment *)
Global`
SomeNamespace`Foo
f[x_, y__, 3, z___] := tsneirsnteintie "fosrt" neisnrteiasrn
E + 3
Plus[1,Times[2,3]]
Map[#1 + #2&, SomePairList]
Plus[1.,-1,-1.,-1.0,]
---tokens---
'1' Literal.Number.Integer
' ' Text.Whitespace
'+' Operator
' ' Text.Whitespace
'1' Literal.Number.Integer
' ' Text.Whitespace
'(* This is a comment *)' Comment
'\n' Text.Whitespace
'Global`' Name.Namespace
'\n' Text.Whitespace
'SomeNamespace`' Name.Namespace
'Foo' Name
'\n' Text.Whitespace
'f' Name
'[' Punctuation
'x_' Name.Variable
',' Punctuation
' ' Text.Whitespace
'y__' Name.Variable
',' Punctuation
' ' Text.Whitespace
'3' Literal.Number.Integer
',' Punctuation
' ' Text.Whitespace
'z___' Name.Variable
']' Punctuation
' ' Text.Whitespace
':=' Operator
' ' Text.Whitespace
'tsneirsnteintie' Name
' ' Text.Whitespace
'"fosrt"' Literal.String
' ' Text.Whitespace
'neisnrteiasrn' Name
'\n' Text.Whitespace
'E' Name
' ' Text.Whitespace
'+' Operator
' ' Text.Whitespace
'3' Literal.Number.Integer
'\n' Text.Whitespace
'Plus' Name
'[' Punctuation
'1' Literal.Number.Integer
',' Punctuation
'Times' Name
'[' Punctuation
'2' Literal.Number.Integer
',' Punctuation
'3' Literal.Number.Integer
']' Punctuation
']' Punctuation
'\n' Text.Whitespace
'Map' Name
'[' Punctuation
'#1' Name.Variable
' ' Text.Whitespace
'+' Operator
' ' Text.Whitespace
'#2' Name.Variable
'&' Operator
',' Punctuation
' ' Text.Whitespace
'SomePairList' Name
']' Punctuation
'\n' Text.Whitespace
'Plus' Name
'[' Punctuation
'1.' Literal.Number.Float
',' Punctuation
'-1' Literal.Number.Integer
',' Punctuation
'-1.' Literal.Number.Float
',' Punctuation
'-1.0' Literal.Number.Float
',' Punctuation
']' Punctuation
'\n' Text.Whitespace
|
