Move test_examplefiles to new tests/lexers scheme.examplefiles

5 files changed, 1208 insertions, 0 deletions

diff --git a/tests/lexers/js/example.txt b/tests/lexers/js/example.txt
new file mode 100644
index 00000000..9f894664
--- /dev/null
+++ b/tests/lexers/js/example.txt
@@ -0,0 +1,175 @@
+---input---
+// regex
+
+blah(/abc/);
+x = /abc/;
+x = /abc/.match;
+
+// math
+
+blah(1/2); //comment
+x = 1 / 2 / 3;
+x = 1/1/.1;
+
+// broken
+
+x=/1/;
+x=1/a/g;
+x=a/a/g;
+
+// real-world
+
+var x = 1/(1+Math.sqrt(sum)); // convert to number between 1-0
+return Math.round((num / den) * 100)/100;
+
+---tokens---
+''            Text
+'// regex\n'  Comment.Single
+
+'\n'          Text
+
+'blah'        Name.Other
+'('           Punctuation
+'/abc/'       Literal.String.Regex
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/abc/'       Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/abc/'       Literal.String.Regex
+'.'           Punctuation
+'match'       Name.Other
+';'           Punctuation
+'\n\n'        Text
+
+'// math\n'   Comment.Single
+
+'\n'          Text
+
+'blah'        Name.Other
+'('           Punctuation
+'1'           Literal.Number.Float
+'/'           Operator
+'2'           Literal.Number.Float
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'//comment\n' Comment.Single
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'1'           Literal.Number.Float
+' '           Text
+'/'           Operator
+' '           Text
+'2'           Literal.Number.Float
+' '           Text
+'/'           Operator
+' '           Text
+'3'           Literal.Number.Float
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'1'           Literal.Number.Float
+'/'           Operator
+'1'           Literal.Number.Float
+'/'           Operator
+'.1'          Literal.Number.Float
+';'           Punctuation
+'\n\n'        Text
+
+'// broken\n' Comment.Single
+
+'\n'          Text
+
+'x'           Name.Other
+'='           Operator
+'/1/'         Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+'='           Operator
+'1'           Literal.Number.Float
+'/'           Operator
+'a'           Name.Other
+'/'           Operator
+'g'           Name.Other
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+'='           Operator
+'a'           Name.Other
+'/'           Operator
+'a'           Name.Other
+'/'           Operator
+'g'           Name.Other
+';'           Punctuation
+'\n\n'        Text
+
+'// real-world\n' Comment.Single
+
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'1'           Literal.Number.Float
+'/'           Operator
+'('           Punctuation
+'1'           Literal.Number.Float
+'+'           Operator
+'Math'        Name.Builtin
+'.'           Punctuation
+'sqrt'        Name.Other
+'('           Punctuation
+'sum'         Name.Other
+')'           Punctuation
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'// convert to number between 1-0\n' Comment.Single
+
+'return'      Keyword
+' '           Text
+'Math'        Name.Builtin
+'.'           Punctuation
+'round'       Name.Other
+'('           Punctuation
+'('           Punctuation
+'num'         Name.Other
+' '           Text
+'/'           Operator
+' '           Text
+'den'         Name.Other
+')'           Punctuation
+' '           Text
+'*'           Operator
+' '           Text
+'100'         Literal.Number.Float
+')'           Punctuation
+'/'           Operator
+'100'         Literal.Number.Float
+';'           Punctuation
+'\n'          Text
diff --git a/tests/lexers/js/example2.txt b/tests/lexers/js/example2.txt
new file mode 100644
index 00000000..003e4cd1
--- /dev/null
+++ b/tests/lexers/js/example2.txt
@@ -0,0 +1,38 @@
+---input---
+var école;
+var sinθ;
+var เมือง;
+var a\u1234b;
+
+var nbsp;
+
+---tokens---
+'var'         Keyword.Declaration
+' '           Text
+'école'       Name.Other
+';'           Punctuation
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'sinθ'        Name.Other
+';'           Punctuation
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'เมือง'       Name.Other
+';'           Punctuation
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'a\\u1234b'   Name.Other
+';'           Punctuation
+'\n\n'        Text
+
+'var'         Keyword.Declaration
+'\u2008'      Text
+'nbsp'        Name.Other
+';'           Punctuation
+'\n'          Text
diff --git a/tests/lexers/js/example3.txt b/tests/lexers/js/example3.txt
new file mode 100644
index 00000000..ca3fc3d9
--- /dev/null
+++ b/tests/lexers/js/example3.txt
@@ -0,0 +1,532 @@
+---input---
+/regexp/.test(foo) || x = [/regexp/,/regexp/, /regexp/, // comment
+// comment
+/regexp/];
+if (/regexp/.test(string))
+{/regexp/.test(string);};
+x =/regexp/;
+x = /regexp/;
+if (0</regexp/.exec(string) || 1>/regexp/.exec(string))
+x = { u:/regexp/, v: /regexp/ };
+foo();/regexp/.test(string); /regexp/.test(string);
+if (!/regexp/) foobar();
+x = u %/regexp/.exec(string) */regexp/.exec(string) / /regexp/.exec(string);
+x = u?/regexp/.exec(string) : v +/regexp/.exec(string) -/regexp/.exec(string);
+a = u^/regexp/.exec(string) &/regexp/.exec(string) |/regexp/.exec(string) +~/regexp/.exec(string);
+x = /regexp/ /* a comment */ ;
+x = /[reg/exp]/;
+x = 4/2/i;
+x = (a == b) ?/* this is a comment */ c : d;
+/// a comment //
+a = /regex//2/1; //syntactically correct, returns NaN
+
+
+
+
+/* original examples */
+
+// regex
+
+blah(/abc/);
+x = /abc/;
+x = /abc/.match;
+
+// math
+
+blah(1/2); //comment
+x = 1 / 2 / 3;
+x = 1/1/.1;
+
+// broken
+
+x=/1/;
+x=1/a/g;
+x=a/a/g;
+
+// real-world
+
+var x = 1/(1+Math.sqrt(sum)); // convert to number between 1-0
+return Math.round((num / den) * 100)/100;
+
+---tokens---
+''            Text
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'test'        Name.Other
+'('           Punctuation
+'foo'         Name.Other
+')'           Punctuation
+' '           Text
+'||'          Operator
+' '           Text
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'['           Punctuation
+'/regexp/'    Literal.String.Regex
+','           Punctuation
+'/regexp/'    Literal.String.Regex
+','           Punctuation
+' '           Text
+'/regexp/'    Literal.String.Regex
+','           Punctuation
+' '           Text
+'// comment\n' Comment.Single
+
+'// comment\n' Comment.Single
+
+'/regexp/'    Literal.String.Regex
+']'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'if'          Keyword
+' '           Text
+'('           Punctuation
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'test'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+')'           Punctuation
+'\n'          Text
+
+'{'           Punctuation
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'test'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+';'           Punctuation
+'}'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+'/regexp/'    Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/regexp/'    Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'if'          Keyword
+' '           Text
+'('           Punctuation
+'0'           Literal.Number.Float
+'<'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'||'          Operator
+' '           Text
+'1'           Literal.Number.Float
+'>'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+')'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'{'           Punctuation
+' '           Text
+'u'           Name.Other
+':'           Operator
+'/regexp/'    Literal.String.Regex
+','           Punctuation
+' '           Text
+'v'           Name.Other
+':'           Operator
+' '           Text
+'/regexp/'    Literal.String.Regex
+' '           Text
+'}'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'foo'         Name.Other
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'test'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'test'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'if'          Keyword
+' '           Text
+'('           Punctuation
+'!'           Operator
+'/regexp/'    Literal.String.Regex
+')'           Punctuation
+' '           Text
+'foobar'      Name.Other
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'u'           Name.Other
+' '           Text
+'%'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'*'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'/'           Operator
+' '           Text
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'u'           Name.Other
+'?'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+':'           Operator
+' '           Text
+'v'           Name.Other
+' '           Text
+'+'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'-'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'a'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'u'           Name.Other
+'^'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'&'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'|'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+' '           Text
+'+'           Operator
+'~'           Operator
+'/regexp/'    Literal.String.Regex
+'.'           Punctuation
+'exec'        Name.Other
+'('           Punctuation
+'string'      Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/regexp/'    Literal.String.Regex
+' '           Text
+'/* a comment */' Comment.Multiline
+' '           Text
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/[reg/exp]/' Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'4'           Literal.Number.Float
+'/'           Operator
+'2'           Literal.Number.Float
+'/'           Operator
+'i'           Name.Other
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'('           Punctuation
+'a'           Name.Other
+' '           Text
+'=='          Operator
+' '           Text
+'b'           Name.Other
+')'           Punctuation
+' '           Text
+'?'           Operator
+'/* this is a comment */' Comment.Multiline
+' '           Text
+'c'           Name.Other
+' '           Text
+':'           Operator
+' '           Text
+'d'           Name.Other
+';'           Punctuation
+'\n'          Text
+
+'/// a comment //\n' Comment.Single
+
+'a'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/regex/'     Literal.String.Regex
+'/'           Operator
+'2'           Literal.Number.Float
+'/'           Operator
+'1'           Literal.Number.Float
+';'           Punctuation
+' '           Text
+'//syntactically correct, returns NaN\n' Comment.Single
+
+'\n\n\n\n'    Text
+
+'/* original examples */' Comment.Multiline
+'\n\n'        Text
+
+'// regex\n'  Comment.Single
+
+'\n'          Text
+
+'blah'        Name.Other
+'('           Punctuation
+'/abc/'       Literal.String.Regex
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/abc/'       Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'/abc/'       Literal.String.Regex
+'.'           Punctuation
+'match'       Name.Other
+';'           Punctuation
+'\n\n'        Text
+
+'// math\n'   Comment.Single
+
+'\n'          Text
+
+'blah'        Name.Other
+'('           Punctuation
+'1'           Literal.Number.Float
+'/'           Operator
+'2'           Literal.Number.Float
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'//comment\n' Comment.Single
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'1'           Literal.Number.Float
+' '           Text
+'/'           Operator
+' '           Text
+'2'           Literal.Number.Float
+' '           Text
+'/'           Operator
+' '           Text
+'3'           Literal.Number.Float
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'1'           Literal.Number.Float
+'/'           Operator
+'1'           Literal.Number.Float
+'/'           Operator
+'.1'          Literal.Number.Float
+';'           Punctuation
+'\n\n'        Text
+
+'// broken\n' Comment.Single
+
+'\n'          Text
+
+'x'           Name.Other
+'='           Operator
+'/1/'         Literal.String.Regex
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+'='           Operator
+'1'           Literal.Number.Float
+'/'           Operator
+'a'           Name.Other
+'/'           Operator
+'g'           Name.Other
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+'='           Operator
+'a'           Name.Other
+'/'           Operator
+'a'           Name.Other
+'/'           Operator
+'g'           Name.Other
+';'           Punctuation
+'\n\n'        Text
+
+'// real-world\n' Comment.Single
+
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'1'           Literal.Number.Float
+'/'           Operator
+'('           Punctuation
+'1'           Literal.Number.Float
+'+'           Operator
+'Math'        Name.Builtin
+'.'           Punctuation
+'sqrt'        Name.Other
+'('           Punctuation
+'sum'         Name.Other
+')'           Punctuation
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'// convert to number between 1-0\n' Comment.Single
+
+'return'      Keyword
+' '           Text
+'Math'        Name.Builtin
+'.'           Punctuation
+'round'       Name.Other
+'('           Punctuation
+'('           Punctuation
+'num'         Name.Other
+' '           Text
+'/'           Operator
+' '           Text
+'den'         Name.Other
+')'           Punctuation
+' '           Text
+'*'           Operator
+' '           Text
+'100'         Literal.Number.Float
+')'           Punctuation
+'/'           Operator
+'100'         Literal.Number.Float
+';'           Punctuation
+'\n'          Text
diff --git a/tests/lexers/js/example4.txt b/tests/lexers/js/example4.txt
new file mode 100644
index 00000000..96e4a719
--- /dev/null
+++ b/tests/lexers/js/example4.txt
@@ -0,0 +1,7 @@
+---input---
+/* There is a BOM at the beginning of this file. */
+
+---tokens---
+''            Text
+'/* There is a BOM at the beginning of this file. */' Comment.Multiline
+'\n'          Text
diff --git a/tests/lexers/js/example5.txt b/tests/lexers/js/example5.txt
new file mode 100644
index 00000000..08e3081d
--- /dev/null
+++ b/tests/lexers/js/example5.txt
@@ -0,0 +1,456 @@
+---input---
+// Most examples from https://github.com/rse/es6-features under MIT license
+const PI = 3.141593;
+
+let callbacks = [];
+
+odds  = evens.map(v => v + 1);
+
+nums.forEach(v => {
+   if (v % 5 === 0)
+       fives.push(v);
+})
+
+function f (x, y, ...a) {
+    return (x + y) * a.length;
+}
+
+var params = [ "hello", true, 7 ];
+var other = [ 1, 2, ...params ]; // [ 1, 2, "hello", true, 7 ]
+f(1, 2, ...params) === 9;
+
+var str = "foo";
+var chars = [ ...str ]; // [ "f", "o", "o" ]
+
+var customer = { name: "Foo" };
+var card = { amount: 7, product: "Bar", unitprice: 42 };
+message = `Hello ${customer.name},
+want to buy ${card.amount} ${card.product} for
+a total of ${card.amount * card.unitprice} bucks?`;
+
+0b111110111 === 503;
+0o767 === 503;
+
+for (let codepoint of "𠮷") console.log(codepoint);
+
+function* ();
+*function();
+yield;
+
+export class Node {
+}
+
+isFinite();
+isNaN();
+isSafeInteger();
+x = new Promise(...a);
+x = new Proxy(...a);
+
+---tokens---
+''            Text
+'// Most examples from https://github.com/rse/es6-features under MIT license\n' Comment.Single
+
+'const'       Keyword.Declaration
+' '           Text
+'PI'          Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'3.141593'    Literal.Number.Float
+';'           Punctuation
+'\n\n'        Text
+
+'let'         Keyword.Declaration
+' '           Text
+'callbacks'   Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'['           Punctuation
+']'           Punctuation
+';'           Punctuation
+'\n\n'        Text
+
+'odds'        Name.Other
+'  '          Text
+'='           Operator
+' '           Text
+'evens'       Name.Other
+'.'           Punctuation
+'map'         Name.Other
+'('           Punctuation
+'v'           Name.Other
+' '           Text
+'=>'          Punctuation
+' '           Text
+'v'           Name.Other
+' '           Text
+'+'           Operator
+' '           Text
+'1'           Literal.Number.Float
+')'           Punctuation
+';'           Punctuation
+'\n\n'        Text
+
+'nums'        Name.Other
+'.'           Punctuation
+'forEach'     Name.Other
+'('           Punctuation
+'v'           Name.Other
+' '           Text
+'=>'          Punctuation
+' '           Text
+'{'           Punctuation
+'\n   '       Text
+'if'          Keyword
+' '           Text
+'('           Punctuation
+'v'           Name.Other
+' '           Text
+'%'           Operator
+' '           Text
+'5'           Literal.Number.Float
+' '           Text
+'==='         Operator
+' '           Text
+'0'           Literal.Number.Float
+')'           Punctuation
+'\n       '   Text
+'fives'       Name.Other
+'.'           Punctuation
+'push'        Name.Other
+'('           Punctuation
+'v'           Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'}'           Punctuation
+')'           Punctuation
+'\n\n'        Text
+
+'function'    Keyword.Declaration
+' '           Text
+'f'           Name.Other
+' '           Text
+'('           Punctuation
+'x'           Name.Other
+','           Punctuation
+' '           Text
+'y'           Name.Other
+','           Punctuation
+' '           Text
+'...'         Punctuation
+'a'           Name.Other
+')'           Punctuation
+' '           Text
+'{'           Punctuation
+'\n    '      Text
+'return'      Keyword
+' '           Text
+'('           Punctuation
+'x'           Name.Other
+' '           Text
+'+'           Operator
+' '           Text
+'y'           Name.Other
+')'           Punctuation
+' '           Text
+'*'           Operator
+' '           Text
+'a'           Name.Other
+'.'           Punctuation
+'length'      Name.Other
+';'           Punctuation
+'\n'          Text
+
+'}'           Punctuation
+'\n\n'        Text
+
+'var'         Keyword.Declaration
+' '           Text
+'params'      Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'['           Punctuation
+' '           Text
+'"hello"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'true'        Keyword.Constant
+','           Punctuation
+' '           Text
+'7'           Literal.Number.Float
+' '           Text
+']'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'other'       Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'['           Punctuation
+' '           Text
+'1'           Literal.Number.Float
+','           Punctuation
+' '           Text
+'2'           Literal.Number.Float
+','           Punctuation
+' '           Text
+'...'         Punctuation
+'params'      Name.Other
+' '           Text
+']'           Punctuation
+';'           Punctuation
+' '           Text
+'// [ 1, 2, "hello", true, 7 ]\n' Comment.Single
+
+'f'           Name.Other
+'('           Punctuation
+'1'           Literal.Number.Float
+','           Punctuation
+' '           Text
+'2'           Literal.Number.Float
+','           Punctuation
+' '           Text
+'...'         Punctuation
+'params'      Name.Other
+')'           Punctuation
+' '           Text
+'==='         Operator
+' '           Text
+'9'           Literal.Number.Float
+';'           Punctuation
+'\n\n'        Text
+
+'var'         Keyword.Declaration
+' '           Text
+'str'         Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'"foo"'       Literal.String.Double
+';'           Punctuation
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'chars'       Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'['           Punctuation
+' '           Text
+'...'         Punctuation
+'str'         Name.Other
+' '           Text
+']'           Punctuation
+';'           Punctuation
+' '           Text
+'// [ "f", "o", "o" ]\n' Comment.Single
+
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'customer'    Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'{'           Punctuation
+' '           Text
+'name'        Name.Other
+':'           Operator
+' '           Text
+'"Foo"'       Literal.String.Double
+' '           Text
+'}'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'var'         Keyword.Declaration
+' '           Text
+'card'        Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'{'           Punctuation
+' '           Text
+'amount'      Name.Other
+':'           Operator
+' '           Text
+'7'           Literal.Number.Float
+','           Punctuation
+' '           Text
+'product'     Name.Other
+':'           Operator
+' '           Text
+'"Bar"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'unitprice'   Name.Other
+':'           Operator
+' '           Text
+'42'          Literal.Number.Float
+' '           Text
+'}'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'message'     Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'`'           Literal.String.Backtick
+'Hello '      Literal.String.Backtick
+'${'          Literal.String.Interpol
+'customer'    Name.Other
+'.'           Punctuation
+'name'        Name.Other
+'}'           Literal.String.Interpol
+',\nwant to buy ' Literal.String.Backtick
+'${'          Literal.String.Interpol
+'card'        Name.Other
+'.'           Punctuation
+'amount'      Name.Other
+'}'           Literal.String.Interpol
+' '           Literal.String.Backtick
+'${'          Literal.String.Interpol
+'card'        Name.Other
+'.'           Punctuation
+'product'     Name.Other
+'}'           Literal.String.Interpol
+' for\na total of ' Literal.String.Backtick
+'${'          Literal.String.Interpol
+'card'        Name.Other
+'.'           Punctuation
+'amount'      Name.Other
+' '           Text
+'*'           Operator
+' '           Text
+'card'        Name.Other
+'.'           Punctuation
+'unitprice'   Name.Other
+'}'           Literal.String.Interpol
+' bucks?'     Literal.String.Backtick
+'`'           Literal.String.Backtick
+';'           Punctuation
+'\n\n'        Text
+
+'0b111110111' Literal.Number.Bin
+' '           Text
+'==='         Operator
+' '           Text
+'503'         Literal.Number.Float
+';'           Punctuation
+'\n'          Text
+
+'0o767'       Literal.Number.Oct
+' '           Text
+'==='         Operator
+' '           Text
+'503'         Literal.Number.Float
+';'           Punctuation
+'\n\n'        Text
+
+'for'         Keyword
+' '           Text
+'('           Punctuation
+'let'         Keyword.Declaration
+' '           Text
+'codepoint'   Name.Other
+' '           Text
+'of'          Keyword
+' '           Text
+'"𠮷"'         Literal.String.Double
+')'           Punctuation
+' '           Text
+'console'     Name.Other
+'.'           Punctuation
+'log'         Name.Other
+'('           Punctuation
+'codepoint'   Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n\n'        Text
+
+'function'    Keyword.Declaration
+'*'           Operator
+' '           Text
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'*'           Operator
+'function'    Keyword.Declaration
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'yield'       Keyword
+';'           Punctuation
+'\n\n'        Text
+
+'export'      Keyword
+' '           Text
+'class'       Keyword.Declaration
+' '           Text
+'Node'        Name.Other
+' '           Text
+'{'           Punctuation
+'\n'          Text
+
+'}'           Punctuation
+'\n\n'        Text
+
+'isFinite'    Name.Builtin
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'isNaN'       Name.Builtin
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'isSafeInteger' Name.Builtin
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'new'         Keyword
+' '           Text
+'Promise'     Name.Builtin
+'('           Punctuation
+'...'         Punctuation
+'a'           Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text
+
+'x'           Name.Other
+' '           Text
+'='           Operator
+' '           Text
+'new'         Keyword
+' '           Text
+'Proxy'       Name.Builtin
+'('           Punctuation
+'...'         Punctuation
+'a'           Name.Other
+')'           Punctuation
+';'           Punctuation
+'\n'          Text