Tests for the improved SQL scanner.

8 files changed, 195 insertions, 95 deletions

diff --git a/test/scanners/sql/create_tables.expected.raydebug b/test/scanners/sql/create_tables.expected.raydebug
index e93a8a1..a4b7f47 100644
--- a/test/scanners/sql/create_tables.expected.raydebug
+++ b/test/scanners/sql/create_tables.expected.raydebug
@@ -15,18 +15,18 @@ comment(--You should have received a copy of the GNU General Public License)
 comment(--along with this program.  If not, see <http://www.gnu.org/licenses/>.)
 
 comment(-- Comment: Drop table)
-reserved(DROP) reserved(TABLE) reserved(IF) reserved(EXISTS) string<delimiter(`)content(general_lookups)delimiter(`)>operator(;)
+class(DROP) type(TABLE) reserved(IF) reserved(EXISTS) string<delimiter(`)content(general_lookups)delimiter(`)>operator(;)
 
 comment(-- Create table)
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(general_lookups)delimiter(`)> operator(()
+class(CREATE) type(TABLE) string<delimiter(`)content(general_lookups)delimiter(`)> operator(()
   string<delimiter(`)content(name)delimiter(`)> pre_type(varchar)operator(()integer(255)operator(\)) directive(default) pre_constant(NULL)
 operator(\)) reserved(ENGINE)operator(=)ident(InnoDB) directive(DEFAULT) directive(CHARSET)operator(=)ident(latin1)operator(;)
 
 comment(-- Drop table again)
-reserved(DROP) reserved(TABLE) reserved(IF) reserved(EXISTS) ident(customer)operator(;)
+class(DROP) type(TABLE) reserved(IF) reserved(EXISTS) ident(customer)operator(;)
 
 comment(-- Create customers)
-reserved(CREATE) reserved(TABLE) ident(customer) operator(()
+class(CREATE) type(TABLE) ident(customer) operator(()
    ident(first_name) pre_type(char)operator(()integer(50)operator(\))operator(,)
    ident(last_name) pre_type(char)operator(()integer(50)operator(\))operator(,)
    ident(address) pre_type(char)operator(()integer(50)operator(\))operator(,)
@@ -38,7 +38,7 @@ reserved(CREATE) reserved(TABLE) ident(customer) operator(()
 operator(\))
 
 comment(-- Create business)
-reserved(CREATE) reserved(TABLE) ident(business) operator(()
+class(CREATE) type(TABLE) ident(business) operator(()
    ident(compant_name) pre_type(char)operator(()integer(50)operator(\))operator(,)
    ident(address) pre_type(char)operator(()integer(50)operator(\)) directive(default) string<delimiter(')content(Address Unknown)delimiter(')>operator(,) comment(-- Oohh, defaults..)
    ident(city) pre_type(char)operator(()integer(50)operator(\)) directive(default) string<delimiter(')content(Adelaide)delimiter(')>operator(,)
@@ -47,9 +47,9 @@ operator(\))
 
 comment(-- Some random table)
 
-reserved(DROP) reserved(TABLE) reserved(IF) reserved(EXISTS) ident(customer_statuses)operator(;)
+class(DROP) type(TABLE) reserved(IF) reserved(EXISTS) ident(customer_statuses)operator(;)
 
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(customer_statuses)delimiter(`)> operator(()
+class(CREATE) type(TABLE) string<delimiter(`)content(customer_statuses)delimiter(`)> operator(()
    comment(-- Auto incrementing IDs)
   string<delimiter(`)content(id)delimiter(`)> pre_type(smallint)operator(()integer(6)operator(\)) pre_type(unsigned) reserved(NOT) pre_constant(NULL) directive(auto_increment)operator(,)
   string<delimiter(`)content(customer_id)delimiter(`)> pre_type(int)operator(()integer(10)operator(\)) pre_type(unsigned) reserved(NOT) pre_constant(NULL) directive(default) string<delimiter(')content(0)delimiter(')>operator(,)
@@ -61,12 +61,12 @@ reserved(CREATE) reserved(TABLE) string<delimiter(`)content(customer_statuses)de
 operator(\)) reserved(ENGINE)operator(=)ident(InnoDB) directive(DEFAULT) directive(CHARSET)operator(=)ident(latin1)operator(;)
 
 comment(-- Try creating an index.)
-reserved(CREATE) reserved(INDEX) ident(customer_status_status_id) reserved(ON) ident(customer_statuses) operator(()ident(status_id)operator(\))
+class(CREATE) type(INDEX) ident(customer_status_status_id) reserved(ON) ident(customer_statuses) operator(()ident(status_id)operator(\))
 
 comment(/* Now lets try and make a really big table */)
 
-reserved(DROP) reserved(TABLE) reserved(IF) reserved(EXISTS) string<delimiter(`)content(legacy_clients)delimiter(`)>operator(;)
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(legacy_clients)delimiter(`)> operator(()
+class(DROP) type(TABLE) reserved(IF) reserved(EXISTS) string<delimiter(`)content(legacy_clients)delimiter(`)>operator(;)
+class(CREATE) type(TABLE) string<delimiter(`)content(legacy_clients)delimiter(`)> operator(()
   string<delimiter(`)content(id)delimiter(`)> pre_type(int)operator(()integer(10)operator(\)) pre_type(unsigned) reserved(NOT) pre_constant(NULL) directive(auto_increment)operator(,)
   string<delimiter(`)content(client_code)delimiter(`)> pre_type(varchar)operator(()integer(15)operator(\)) directive(default) pre_constant(NULL)operator(,)
   string<delimiter(`)content(first_name)delimiter(`)> pre_type(varchar)operator(()integer(20)operator(\)) reserved(NOT) pre_constant(NULL) directive(default) string<delimiter(')delimiter(')>operator(,)
diff --git a/test/scanners/sql/maintenance.expected.raydebug b/test/scanners/sql/maintenance.expected.raydebug
index 440564e..6cfb72a 100644
--- a/test/scanners/sql/maintenance.expected.raydebug
+++ b/test/scanners/sql/maintenance.expected.raydebug
@@ -14,11 +14,11 @@ comment(--GNU General Public License for more details.)
 comment(--You should have received a copy of the GNU General Public License)
 comment(--along with this program.  If not, see <http://www.gnu.org/licenses/>.)
 
-reserved(INSERT) reserved(INTO) ident(users) operator(()ident(first_name)operator(,) ident(last_name)operator(\)) reserved(VALUES) operator(()string<delimiter(')content(John)delimiter(')>operator(,) string<delimiter(')content(Doe)delimiter(')>operator(\))operator(;)
+class(INSERT) class(INTO) ident(users) operator(()ident(first_name)operator(,) ident(last_name)operator(\)) reserved(VALUES) operator(()string<delimiter(')content(John)delimiter(')>operator(,) string<delimiter(')content(Doe)delimiter(')>operator(\))operator(;)
 
-reserved(INSERT) reserved(INTO) ident(users) operator(()ident(first_name)operator(,) ident(last_name)operator(\)) reserved(VALUES) operator(()string<delimiter(")content(John)delimiter(")>operator(,) string<delimiter(")content(Doe)delimiter(")>operator(\))operator(;)
+class(INSERT) class(INTO) ident(users) operator(()ident(first_name)operator(,) ident(last_name)operator(\)) reserved(VALUES) operator(()string<delimiter(")content(John)delimiter(")>operator(,) string<delimiter(")content(Doe)delimiter(")>operator(\))operator(;)
 
-reserved(UPDATE) ident(users) reserved(SET) ident(first_name) operator(=) string<delimiter(')content(Keith)delimiter(')> reserved(WHERE) ident(first_name) operator(=) string<delimiter(')content(JOHN)delimiter(')>operator(;)
+class(UPDATE) ident(users) class(SET) ident(first_name) operator(=) string<delimiter(')content(Keith)delimiter(')> reserved(WHERE) ident(first_name) operator(=) string<delimiter(')content(JOHN)delimiter(')>operator(;)
 
-reserved(DELETE) reserved(FROM) ident(users) reserved(WHERE) ident(first_name) operator(=) string<delimiter(')content(Keith)delimiter(')>operator(;)
+class(DELETE) reserved(FROM) ident(users) reserved(WHERE) ident(first_name) operator(=) string<delimiter(')content(Keith)delimiter(')>operator(;)
 
diff --git a/test/scanners/sql/mysql-comments.expected.raydebug b/test/scanners/sql/mysql-comments.expected.raydebug
new file mode 100644
index 0000000..38f738c
--- /dev/null
+++ b/test/scanners/sql/mysql-comments.expected.raydebug
@@ -0,0 +1,2 @@
+class(SELECT) directive(/*! STRAIGHT_JOIN */) ident(col1) reserved(FROM) ident(table1)operator(,)ident(table2) reserved(WHERE) operator(.)operator(.)operator(.)
+class(CREATE) directive(/*!32302 TEMPORARY */) type(TABLE) ident(t) operator(()ident(a) pre_type(INT)operator(\))operator(;)
diff --git a/test/scanners/sql/mysql-comments.in.sql b/test/scanners/sql/mysql-comments.in.sql
new file mode 100644
index 0000000..fe7cd83
--- /dev/null
+++ b/test/scanners/sql/mysql-comments.in.sql
@@ -0,0 +1,2 @@
+SELECT /*! STRAIGHT_JOIN */ col1 FROM table1,table2 WHERE ...
+CREATE /*!32302 TEMPORARY */ TABLE t (a INT);
diff --git a/test/scanners/sql/norwegian.expected.raydebug b/test/scanners/sql/norwegian.expected.raydebug
new file mode 100644
index 0000000..1222709
--- /dev/null
+++ b/test/scanners/sql/norwegian.expected.raydebug
@@ -0,0 +1,48 @@
+class(select) integer(1) reserved(from)comment(-- Skapa en ny tabell med de endast de språkkoder vi accepterar)
+class(CREATE) type(TABLE) ident(public)operator(.)ident(languages) operator(()
+    ident(language_code) pre_type(char)operator(()integer(2)operator(\)) reserved(PRIMARY) reserved(KEY)
+operator(\))operator(;)
+class(INSERT) class(INTO) ident(public)operator(.)ident(languages) reserved(VALUES) operator(()string<delimiter(')content(sv)delimiter(')>operator(\))operator(,)operator(()string<delimiter(')content(de)delimiter(')>operator(\))operator(,)operator(()string<delimiter(')content(da)delimiter(')>operator(\))operator(,)operator(()string<delimiter(')content(no)delimiter(')>operator(\))operator(,)operator(()string<delimiter(')content(fi)delimiter(')>operator(\))operator(;)
+class(COMMENT) reserved(ON) type(COLUMN) ident(public)operator(.)ident(languages)operator(.)ident(language_code) reserved(IS) string<delimiter(')content(Codes according to ISO 639)delimiter(')>operator(;)
+class(GRANT) class(SELECT) reserved(ON) ident(public)operator(.)ident(languages) reserved(TO) reserved(GROUP) ident(readonly)operator(;)
+class(GRANT) reserved(ALL)    reserved(ON) ident(public)operator(.)ident(languages) reserved(TO) reserved(GROUP) ident(readwrite)operator(;)
+
+comment(-- Lägg till en ny column i color_description som hanterar language_code instället för)
+comment(-- locale som idag.)
+comment(--   1. Lägg till kolumnen.)
+comment(--   2. Fyll den med data från locale kolumnen.)
+comment(--   3. Ändra så att columnen inte får vara tom)
+comment(--   4. Lägg till constraint mot tidigare skapade tabellen languages.)
+comment(--   5. Sätt kommentar på locale kolumnen om att den inte bör användas.)
+class(ALTER) type(TABLE) ident(products)operator(.)ident(color_description) class(ADD) type(COLUMN) ident(language_code) pre_type(char)operator(()integer(2)operator(\))operator(;)
+class(UPDATE) ident(products)operator(.)ident(color_description)
+class(SET) ident(language_code) operator(=) operator(()reserved(CASE) reserved(WHEN) ident(locale) reserved(like) string<delimiter(')content(ty%)delimiter(')>
+                        reserved(THEN) string<delimiter(')content(de)delimiter(')>
+                        reserved(ELSE) predefined(SUBSTRING)operator(()ident(locale) reserved(FROM) integer(1) reserved(FOR) integer(2)operator(\)) reserved(END)operator(\))operator(;)
+class(ALTER) type(TABLE) ident(products)operator(.)ident(color_description) class(ALTER) type(COLUMN) ident(language_code) class(SET) reserved(NOT) pre_constant(NULL)operator(;)
+class(ALTER) type(TABLE) ident(products)operator(.)ident(color_description)
+    class(ADD) reserved(CONSTRAINT) ident(products_color_description_lang_ref)
+    reserved(FOREIGN) reserved(KEY) operator(()ident(language_code)operator(\)) reserved(REFERENCES) ident(public)operator(.)ident(languages)operator(()ident(language_code)operator(\))operator(;)
+class(COMMENT) reserved(ON) type(COLUMN) ident(products)operator(.)ident(color_description)operator(.)ident(locale)
+reserved(IS) string<delimiter(')content(<¡> PAHSE OUT, use language_code instead)delimiter(')>operator(;)
+
+
+
+comment(-- Lägg till en ny column i frame_description som hanterar language_code instället för)
+comment(-- locale som idag.)
+comment(--   1. Lägg till kolumnen.)
+comment(--   2. Fyll den med data från locale kolumnen.)
+comment(--   3. Ändra så att columnen inte får vara tom)
+comment(--   4. Lägg till constraint mot tidigare skapade tabellen languages.)
+comment(--   5. Sätt kommentar på locale kolumnen om att den inte bör användas.)
+class(ALTER) type(TABLE) ident(products)operator(.)ident(frame_description) class(ADD) type(COLUMN) ident(language_code) pre_type(char)operator(()integer(2)operator(\))operator(;)
+class(UPDATE) ident(products)operator(.)ident(frame_description)
+class(SET) ident(language_code) operator(=) operator(()reserved(CASE) reserved(WHEN) ident(locale) reserved(like) string<delimiter(')content(ty%)delimiter(')>
+                        reserved(THEN) string<delimiter(')content(de)delimiter(')>
+                        reserved(ELSE) predefined(SUBSTRING)operator(()ident(locale) reserved(FROM) integer(1) reserved(FOR) integer(2)operator(\)) reserved(END)operator(\))operator(;)
+class(ALTER) type(TABLE) ident(products)operator(.)ident(frame_description) class(ALTER) type(COLUMN) ident(language_code) class(SET) reserved(NOT) pre_constant(NULL)operator(;)
+class(ALTER) type(TABLE) ident(products)operator(.)ident(frame_description)
+    class(ADD) reserved(CONSTRAINT) ident(products_frame_description_lang_ref)
+    reserved(FOREIGN) reserved(KEY) operator(()ident(language_code)operator(\)) reserved(REFERENCES) ident(public)operator(.)ident(languages)operator(()ident(language_code)operator(\))operator(;)
+class(COMMENT) reserved(ON) type(COLUMN) ident(products)operator(.)ident(frame_description)operator(.)ident(locale)
+reserved(IS) string<delimiter(')content(<¡> PAHSE OUT, use language_code instead)delimiter(')>operator(;)
diff --git a/test/scanners/sql/norwegian.in.sql b/test/scanners/sql/norwegian.in.sql
new file mode 100644
index 0000000..b1241b8
--- /dev/null
+++ b/test/scanners/sql/norwegian.in.sql
@@ -0,0 +1,48 @@
+select 1 from-- Skapa en ny tabell med de endast de språkkoder vi accepterar
+CREATE TABLE public.languages (
+    language_code char(2) PRIMARY KEY
+);
+INSERT INTO public.languages VALUES ('sv'),('de'),('da'),('no'),('fi');
+COMMENT ON COLUMN public.languages.language_code IS 'Codes according to ISO 639';
+GRANT SELECT ON public.languages TO GROUP readonly;
+GRANT ALL    ON public.languages TO GROUP readwrite;
+
+-- Lägg till en ny column i color_description som hanterar language_code instället för
+-- locale som idag.
+--   1. Lägg till kolumnen.
+--   2. Fyll den med data från locale kolumnen.
+--   3. Ändra så att columnen inte får vara tom
+--   4. Lägg till constraint mot tidigare skapade tabellen languages.
+--   5. Sätt kommentar på locale kolumnen om att den inte bör användas.
+ALTER TABLE products.color_description ADD COLUMN language_code char(2);
+UPDATE products.color_description
+SET language_code = (CASE WHEN locale like 'ty%'
+                        THEN 'de'
+                        ELSE SUBSTRING(locale FROM 1 FOR 2) END);
+ALTER TABLE products.color_description ALTER COLUMN language_code SET NOT NULL;
+ALTER TABLE products.color_description
+    ADD CONSTRAINT products_color_description_lang_ref
+    FOREIGN KEY (language_code) REFERENCES public.languages(language_code);
+COMMENT ON COLUMN products.color_description.locale
+IS '<¡> PAHSE OUT, use language_code instead';
+
+
+
+-- Lägg till en ny column i frame_description som hanterar language_code instället för
+-- locale som idag.
+--   1. Lägg till kolumnen.
+--   2. Fyll den med data från locale kolumnen.
+--   3. Ändra så att columnen inte får vara tom
+--   4. Lägg till constraint mot tidigare skapade tabellen languages.
+--   5. Sätt kommentar på locale kolumnen om att den inte bör användas.
+ALTER TABLE products.frame_description ADD COLUMN language_code char(2);
+UPDATE products.frame_description
+SET language_code = (CASE WHEN locale like 'ty%'
+                        THEN 'de'
+                        ELSE SUBSTRING(locale FROM 1 FOR 2) END);
+ALTER TABLE products.frame_description ALTER COLUMN language_code SET NOT NULL;
+ALTER TABLE products.frame_description
+    ADD CONSTRAINT products_frame_description_lang_ref
+    FOREIGN KEY (language_code) REFERENCES public.languages(language_code);
+COMMENT ON COLUMN products.frame_description.locale
+IS '<¡> PAHSE OUT, use language_code instead';
diff --git a/test/scanners/sql/reference.expected.raydebug b/test/scanners/sql/reference.expected.raydebug
index 5251519..443c59b 100644
--- a/test/scanners/sql/reference.expected.raydebug
+++ b/test/scanners/sql/reference.expected.raydebug
@@ -4,106 +4,106 @@ comment(#   http://dev.mysql.com/doc/refman/5.1/en/language-structure.html)
 comment(# unless stated otherwise.)
 
 comment(# strings)
-reserved(SELECT) string<delimiter(')content(a string)delimiter(')>operator(;)
-reserved(SELECT) string<delimiter(")content(another string)delimiter(")>operator(;)
+class(SELECT) string<delimiter(')content(a string)delimiter(')>operator(;)
+class(SELECT) string<delimiter(")content(another string)delimiter(")>operator(;)
 
-reserved(SELECT) string<modifier(_latin1)delimiter(')content(string)delimiter(')>operator(;)
-reserved(SELECT) string<modifier(_latin1)delimiter(')content(string)delimiter(')> reserved(COLLATE) ident(latin1_danish_ci)operator(;)
+class(SELECT) string<modifier(_latin1)delimiter(')content(string)delimiter(')>operator(;)
+class(SELECT) string<modifier(_latin1)delimiter(')content(string)delimiter(')> reserved(COLLATE) ident(latin1_danish_ci)operator(;)
 
-reserved(SELECT) string<modifier(N)delimiter(')content(some text)delimiter(')>operator(;)
-reserved(SELECT) string<modifier(n)delimiter(')content(some text)delimiter(')>operator(;)
-reserved(SELECT) string<modifier(_utf8)delimiter(')content(some text)delimiter(')>operator(;)
+class(SELECT) string<modifier(N)delimiter(')content(some text)delimiter(')>operator(;)
+class(SELECT) string<modifier(n)delimiter(')content(some text)delimiter(')>operator(;)
+class(SELECT) string<modifier(_utf8)delimiter(')content(some text)delimiter(')>operator(;)
 
-reserved(SELECT) string<delimiter(")char(\\0)char(\\')char(\\")content(''"")char(\\b)char(\\n)char(\\r)char(\\t)char(\\Z)char(\\\\)char(\\%)char(\\_)delimiter(")>operator(;)  comment(# ")
-reserved(SELECT) string<delimiter(')char(\\0)char(\\')char(\\")content(''"")char(\\b)char(\\n)char(\\r)char(\\t)char(\\Z)char(\\\\)char(\\%)char(\\_)delimiter(')>operator(;)  comment(# ')
+class(SELECT) string<delimiter(")char(\\0)char(\\')char(\\")content(''"")char(\\b)char(\\n)char(\\r)char(\\t)char(\\Z)char(\\\\)char(\\%)char(\\_)delimiter(")>operator(;)  comment(# ")
+class(SELECT) string<delimiter(')char(\\0)char(\\')char(\\")content(''"")char(\\b)char(\\n)char(\\r)char(\\t)char(\\Z)char(\\\\)char(\\%)char(\\_)delimiter(')>operator(;)  comment(# ')
 
-reserved(SELECT) string<delimiter(")char(\\B)char(\\x)delimiter(")>operator(;)  comment(# ")
-reserved(SELECT) string<delimiter(')char(\\B)char(\\x)delimiter(')>operator(;)  comment(# ')
+class(SELECT) string<delimiter(")char(\\B)char(\\x)delimiter(")>operator(;)  comment(# ")
+class(SELECT) string<delimiter(')char(\\B)char(\\x)delimiter(')>operator(;)  comment(# ')
 
-reserved(SELECT) string<delimiter(')content(hello)delimiter(')>operator(,) string<delimiter(')content("hello")delimiter(')>operator(,) string<delimiter(')content(""hello"")delimiter(')>operator(,) string<delimiter(')content(hel''lo)delimiter(')>operator(,) string<delimiter(')char(\\')content(hello)delimiter(')>operator(;)  comment(-- ')
-reserved(SELECT) string<delimiter(")content(hello)delimiter(")>operator(,) string<delimiter(")content('hello')delimiter(")>operator(,) string<delimiter(")content(''hello'')delimiter(")>operator(,) string<delimiter(")content(hel""lo)delimiter(")>operator(,) string<delimiter(")char(\\")content(hello)delimiter(")>operator(;)  comment(-- ")
+class(SELECT) string<delimiter(')content(hello)delimiter(')>operator(,) string<delimiter(')content("hello")delimiter(')>operator(,) string<delimiter(')content(""hello"")delimiter(')>operator(,) string<delimiter(')content(hel''lo)delimiter(')>operator(,) string<delimiter(')char(\\')content(hello)delimiter(')>operator(;)  comment(-- ')
+class(SELECT) string<delimiter(")content(hello)delimiter(")>operator(,) string<delimiter(")content('hello')delimiter(")>operator(,) string<delimiter(")content(''hello'')delimiter(")>operator(,) string<delimiter(")content(hel""lo)delimiter(")>operator(,) string<delimiter(")char(\\")content(hello)delimiter(")>operator(;)  comment(-- ")
 
-reserved(SELECT) string<delimiter(')content(This)char(\\n)content(Is)char(\\n)content(Four)char(\\n)content(Lines)delimiter(')>operator(;)
-reserved(SELECT) string<delimiter(')content(disappearing)char(\\ )content(backslash)delimiter(')>operator(;)
+class(SELECT) string<delimiter(')content(This)char(\\n)content(Is)char(\\n)content(Four)char(\\n)content(Lines)delimiter(')>operator(;)
+class(SELECT) string<delimiter(')content(disappearing)char(\\ )content(backslash)delimiter(')>operator(;)
 
 comment(# numbers)
-reserved(select) integer(1221)operator(;)
-reserved(select) integer(0)operator(;)
-reserved(select) operator(-)integer(32)error(:)
+class(select) integer(1221)operator(;)
+class(select) integer(0)operator(;)
+class(select) operator(-)integer(32)error(:)
 
-reserved(select) float(294.42)error(:)
-reserved(select) operator(-)float(32032.6809e+10)operator(;)
-reserved(select) float(148.00)operator(;)
+class(select) float(294.42)error(:)
+class(select) operator(-)float(32032.6809e+10)operator(;)
+class(select) float(148.00)operator(;)
 
-reserved(select) float(10e+10)operator(;)
-reserved(select) float(10e10)operator(;)
+class(select) float(10e+10)operator(;)
+class(select) float(10e10)operator(;)
 
 comment(# hexadecimal)
-reserved(SELECT) string<modifier(X)delimiter(')content(4D7953514C)delimiter(')>operator(;)
-reserved(SELECT) hex(0x0a)operator(+)integer(0)operator(;)
-reserved(SELECT) hex(0x5061756c)operator(;)
-reserved(SELECT) hex(0x41)operator(,) predefined(CAST)operator(()hex(0x41) reserved(AS) pre_type(UNSIGNED)operator(\))operator(;)
-reserved(SELECT) pre_type(HEX)operator(()string<delimiter(')content(cat)delimiter(')>operator(\))operator(;)
-reserved(SELECT) hex(0x636174)operator(;)
-reserved(insert) reserved(into) ident(t) operator(()ident(md5)operator(\)) reserved(values) operator(()hex(0xad65)operator(\))operator(;)
-reserved(SELECT) operator(*) reserved(FROM) ident(SomeTable) reserved(WHERE) ident(BinaryColumn) operator(=) predefined(CAST)operator(() string<modifier(x)delimiter(')content(a0f44ef7a52411de)delimiter(')> reserved(AS) pre_type(BINARY) operator(\))operator(;)
-reserved(select) string<modifier(x)delimiter(')content(000bdddc0e9153f5a93447fc3310f710)delimiter(')>operator(,) string<modifier(x)delimiter(')content(0bdddc0e9153f5a93447fc3310f710)delimiter(')>operator(;)
-
-reserved(SELECT) pre_constant(TRUE)operator(,) pre_constant(true)operator(,) pre_constant(FALSE)operator(,) pre_constant(false)operator(;)
-reserved(SELECT) pre_constant(NULL)operator(,) pre_constant(null)operator(,) pre_constant(nuLL)operator(,) error(\\)ident(N)operator(;)
-reserved(SELECT) error(\\)ident(n)operator(;)  comment(# invalid!)
+class(SELECT) string<modifier(X)delimiter(')content(4D7953514C)delimiter(')>operator(;)
+class(SELECT) hex(0x0a)operator(+)integer(0)operator(;)
+class(SELECT) hex(0x5061756c)operator(;)
+class(SELECT) hex(0x41)operator(,) predefined(CAST)operator(()hex(0x41) reserved(AS) pre_type(UNSIGNED)operator(\))operator(;)
+class(SELECT) pre_type(HEX)operator(()string<delimiter(')content(cat)delimiter(')>operator(\))operator(;)
+class(SELECT) hex(0x636174)operator(;)
+class(insert) class(into) ident(t) operator(()ident(md5)operator(\)) reserved(values) operator(()hex(0xad65)operator(\))operator(;)
+class(SELECT) operator(*) reserved(FROM) ident(SomeTable) reserved(WHERE) ident(BinaryColumn) operator(=) predefined(CAST)operator(() string<modifier(x)delimiter(')content(a0f44ef7a52411de)delimiter(')> reserved(AS) pre_type(BINARY) operator(\))operator(;)
+class(select) string<modifier(x)delimiter(')content(000bdddc0e9153f5a93447fc3310f710)delimiter(')>operator(,) string<modifier(x)delimiter(')content(0bdddc0e9153f5a93447fc3310f710)delimiter(')>operator(;)
+
+class(SELECT) pre_constant(TRUE)operator(,) pre_constant(true)operator(,) pre_constant(FALSE)operator(,) pre_constant(false)operator(;)
+class(SELECT) pre_constant(NULL)operator(,) pre_constant(null)operator(,) pre_constant(nuLL)operator(,) pre_constant(\\N)operator(;)
+class(SELECT) error(\\)ident(n)operator(;)  comment(# invalid!)
 
 comment(# bit-field)
-reserved(CREATE) reserved(TABLE) ident(t) operator(()ident(b) pre_type(BIT)operator(()integer(8)operator(\))operator(\))operator(;)
-reserved(INSERT) reserved(INTO) ident(t) reserved(SET) ident(b) operator(=) string<modifier(b)delimiter(')content(11111111)delimiter(')>operator(;)
-reserved(INSERT) reserved(INTO) ident(t) reserved(SET) ident(b) operator(=) string<modifier(b)delimiter(')content(1010)delimiter(')>operator(;)
-reserved(INSERT) reserved(INTO) ident(t) reserved(SET) ident(b) operator(=) string<modifier(b)delimiter(')content(0101)delimiter(')>operator(;)
-reserved(SELECT) ident(b)operator(+)integer(0)operator(,) pre_type(BIN)operator(()ident(b)operator(+)integer(0)operator(\))operator(,) pre_type(OCT)operator(()ident(b)operator(+)integer(0)operator(\))operator(,) pre_type(HEX)operator(()ident(b)operator(+)integer(0)operator(\)) reserved(FROM) ident(t)operator(;)
+class(CREATE) type(TABLE) ident(t) operator(()ident(b) pre_type(BIT)operator(()integer(8)operator(\))operator(\))operator(;)
+class(INSERT) class(INTO) ident(t) class(SET) ident(b) operator(=) string<modifier(b)delimiter(')content(11111111)delimiter(')>operator(;)
+class(INSERT) class(INTO) ident(t) class(SET) ident(b) operator(=) string<modifier(b)delimiter(')content(1010)delimiter(')>operator(;)
+class(INSERT) class(INTO) ident(t) class(SET) ident(b) operator(=) string<modifier(b)delimiter(')content(0101)delimiter(')>operator(;)
+class(SELECT) ident(b)operator(+)integer(0)operator(,) pre_type(BIN)operator(()ident(b)operator(+)integer(0)operator(\))operator(,) pre_type(OCT)operator(()ident(b)operator(+)integer(0)operator(\))operator(,) pre_type(HEX)operator(()ident(b)operator(+)integer(0)operator(\)) reserved(FROM) ident(t)operator(;)
 
-reserved(SET) variable(@v1) operator(=) string<modifier(b)delimiter(')content(1000001)delimiter(')>operator(;)
-reserved(SET) variable(@v2) operator(=) predefined(CAST)operator(()string<modifier(b)delimiter(')content(1000001)delimiter(')> reserved(AS) pre_type(UNSIGNED)operator(\))operator(,) variable(@v3) operator(=) string<modifier(b)delimiter(')content(1000001)delimiter(')>operator(+)integer(0)operator(;)
-reserved(SELECT) variable(@v1)operator(,) variable(@v2)operator(,) variable(@v3)operator(;)
+class(SET) variable(@v1) operator(=) string<modifier(b)delimiter(')content(1000001)delimiter(')>operator(;)
+class(SET) variable(@v2) operator(=) predefined(CAST)operator(()string<modifier(b)delimiter(')content(1000001)delimiter(')> reserved(AS) pre_type(UNSIGNED)operator(\))operator(,) variable(@v3) operator(=) string<modifier(b)delimiter(')content(1000001)delimiter(')>operator(+)integer(0)operator(;)
+class(SELECT) variable(@v1)operator(,) variable(@v2)operator(,) variable(@v3)operator(;)
 
-reserved(INSERT) reserved(INTO) ident(my_table) operator(()ident(phone)operator(\)) reserved(VALUES) operator(()pre_constant(NULL)operator(\))operator(;)
-reserved(INSERT) reserved(INTO) ident(my_table) operator(()ident(phone)operator(\)) reserved(VALUES) operator(()string<delimiter(')delimiter(')>operator(\))operator(;)
+class(INSERT) class(INTO) ident(my_table) operator(()ident(phone)operator(\)) reserved(VALUES) operator(()pre_constant(NULL)operator(\))operator(;)
+class(INSERT) class(INTO) ident(my_table) operator(()ident(phone)operator(\)) reserved(VALUES) operator(()string<delimiter(')delimiter(')>operator(\))operator(;)
 
 comment(# schema object names)
-reserved(SELECT) operator(*) reserved(FROM) string<delimiter(`)content(select)delimiter(`)> reserved(WHERE) string<delimiter(`)content(select)delimiter(`)>operator(.)ident(id) operator(>) integer(100)operator(;)
+class(SELECT) operator(*) reserved(FROM) string<delimiter(`)content(select)delimiter(`)> reserved(WHERE) string<delimiter(`)content(select)delimiter(`)>operator(.)ident(id) operator(>) integer(100)operator(;)
 
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(a``b)delimiter(`)> operator(()string<delimiter(`)content(c"d)delimiter(`)> pre_type(INT)operator(\))operator(;)
-reserved(SELECT) integer(1) reserved(AS) string<delimiter(`)content(one)delimiter(`)>operator(,) integer(2) reserved(AS) string<delimiter(')content(two)delimiter(')>operator(;)
+class(CREATE) type(TABLE) string<delimiter(`)content(a``b)delimiter(`)> operator(()string<delimiter(`)content(c"d)delimiter(`)> pre_type(INT)operator(\))operator(;)
+class(SELECT) integer(1) reserved(AS) string<delimiter(`)content(one)delimiter(`)>operator(,) integer(2) reserved(AS) string<delimiter(')content(two)delimiter(')>operator(;)
 
-reserved(select) ident(foo) reserved(from) ident(foo)operator(;)
-reserved(select) string<delimiter(`)content(foo)delimiter(`)> reserved(from) ident(foo)operator(;)
-reserved(select) ident(foo)operator(.)ident(bar) reserved(from) ident(foo)operator(;)
-reserved(select) string<delimiter(`)content(foo)delimiter(`)>operator(.)ident(bar) reserved(from) ident(foo)operator(;)
-reserved(select) ident(foo)operator(.)string<delimiter(`)content(bar)delimiter(`)> reserved(from) ident(foo)operator(;)
-reserved(select) string<delimiter(`)content(foo.bar)delimiter(`)> reserved(from) ident(foo)operator(;)
-reserved(select) string<delimiter(`)content(foo)delimiter(`)>operator(.)string<delimiter(`)content(bar)delimiter(`)> reserved(from) ident(foo)operator(;)
+class(select) ident(foo) reserved(from) ident(foo)operator(;)
+class(select) string<delimiter(`)content(foo)delimiter(`)> reserved(from) ident(foo)operator(;)
+class(select) ident(foo)operator(.)ident(bar) reserved(from) ident(foo)operator(;)
+class(select) string<delimiter(`)content(foo)delimiter(`)>operator(.)ident(bar) reserved(from) ident(foo)operator(;)
+class(select) ident(foo)operator(.)string<delimiter(`)content(bar)delimiter(`)> reserved(from) ident(foo)operator(;)
+class(select) string<delimiter(`)content(foo.bar)delimiter(`)> reserved(from) ident(foo)operator(;)
+class(select) string<delimiter(`)content(foo)delimiter(`)>operator(.)string<delimiter(`)content(bar)delimiter(`)> reserved(from) ident(foo)operator(;)
 
 comment(# How to handle ANSI_QUOTES?)
-reserved(CREATE) reserved(TABLE) string<delimiter(")content(test)delimiter(")> operator(()ident(col) pre_type(INT)operator(\))operator(;)
-reserved(SET) ident(sql_mode)operator(=)string<delimiter(')content(ANSI_QUOTES)delimiter(')>operator(;)
-reserved(CREATE) reserved(TABLE) string<delimiter(")content(test)delimiter(")> operator(()ident(col) pre_type(INT)operator(\))operator(;)
+class(CREATE) type(TABLE) string<delimiter(")content(test)delimiter(")> operator(()ident(col) pre_type(INT)operator(\))operator(;)
+class(SET) ident(sql_mode)operator(=)string<delimiter(')content(ANSI_QUOTES)delimiter(')>operator(;)
+class(CREATE) type(TABLE) string<delimiter(")content(test)delimiter(")> operator(()ident(col) pre_type(INT)operator(\))operator(;)
 
 comment(# identifiers)
-reserved(SELECT) operator(*) reserved(FROM) ident(my_table) reserved(WHERE) ident(MY_TABLE)operator(.)ident(col)operator(=)integer(1)operator(;)
-reserved(SHOW) reserved(COLUMNS) reserved(FROM) string<delimiter(`)content(#mysql50#a@b)delimiter(`)>operator(;)
+class(SELECT) operator(*) reserved(FROM) ident(my_table) reserved(WHERE) ident(MY_TABLE)operator(.)ident(col)operator(=)integer(1)operator(;)
+class(SHOW) type(COLUMNS) reserved(FROM) string<delimiter(`)content(#mysql50#a@b)delimiter(`)>operator(;)
 
 comment(# Function Name Parsing and Resolution)
 
 
-reserved(SELECT) predefined(COUNT)operator(()operator(*)operator(\)) reserved(FROM) ident(mytable)operator(;)  comment(-- the first reference to count is a function call)
-reserved(CREATE) reserved(TABLE) predefined(count) operator(()ident(i) pre_type(INT)operator(\))operator(;)  comment(-- whereas the second reference is a table name)
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(count)delimiter(`)>operator(()ident(i) pre_type(INT)operator(\))operator(;)  comment(-- this too)
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(count)delimiter(`)> operator(()ident(i) pre_type(INT)operator(\))operator(;)  comment(-- this too)
+class(SELECT) predefined(COUNT)operator(()operator(*)operator(\)) reserved(FROM) ident(mytable)operator(;)  comment(-- the first reference to count is a function call)
+class(CREATE) type(TABLE) predefined(count) operator(()ident(i) pre_type(INT)operator(\))operator(;)  comment(-- whereas the second reference is a table name)
+class(CREATE) type(TABLE) string<delimiter(`)content(count)delimiter(`)>operator(()ident(i) pre_type(INT)operator(\))operator(;)  comment(-- this too)
+class(CREATE) type(TABLE) string<delimiter(`)content(count)delimiter(`)> operator(()ident(i) pre_type(INT)operator(\))operator(;)  comment(-- this too)
 
 comment(# IGNORE_SPACE)
-reserved(SELECT) predefined(COUNT)operator(()operator(*)operator(\)) reserved(FROM) ident(mytable)operator(;)
-reserved(SELECT) predefined(COUNT) operator(()operator(*)operator(\)) reserved(FROM) ident(mytable)operator(;)
+class(SELECT) predefined(COUNT)operator(()operator(*)operator(\)) reserved(FROM) ident(mytable)operator(;)
+class(SELECT) predefined(COUNT) operator(()operator(*)operator(\)) reserved(FROM) ident(mytable)operator(;)
 
 comment(# reserved words)
-reserved(CREATE) reserved(TABLE) ident(interval) operator(()reserved(begin) pre_type(INT)operator(,) reserved(end) pre_type(INT)operator(\))operator(;)  comment(-- errror)
-reserved(CREATE) reserved(TABLE) string<delimiter(`)content(interval)delimiter(`)> operator(()reserved(begin) pre_type(INT)operator(,) reserved(end) pre_type(INT)operator(\))operator(;)  comment(-- valid)
-reserved(CREATE) reserved(TABLE) ident(mydb)operator(.)ident(interval) operator(()reserved(begin) pre_type(INT)operator(,) reserved(end) pre_type(INT)operator(\))operator(;)  comment(-- valid)
-reserved(SELECT) string<delimiter(`)content(foo)delimiter(`)>operator(,) string<delimiter(`)content(bar)delimiter(`)> reserved(FROM) string<delimiter(`)content(baz)delimiter(`)> reserved(WHERE) string<delimiter(`)content(bal)delimiter(`)> operator(=) string<delimiter(`)content(quiche)delimiter(`)>operator(;)  comment(-- valid)
+class(CREATE) type(TABLE) ident(interval) operator(()reserved(begin) pre_type(INT)operator(,) reserved(end) pre_type(INT)operator(\))operator(;)  comment(-- errror)
+class(CREATE) type(TABLE) string<delimiter(`)content(interval)delimiter(`)> operator(()reserved(begin) pre_type(INT)operator(,) reserved(end) pre_type(INT)operator(\))operator(;)  comment(-- valid)
+class(CREATE) type(TABLE) ident(mydb)operator(.)ident(interval) operator(()reserved(begin) pre_type(INT)operator(,) reserved(end) pre_type(INT)operator(\))operator(;)  comment(-- valid)
+class(SELECT) string<delimiter(`)content(foo)delimiter(`)>operator(,) string<delimiter(`)content(bar)delimiter(`)> reserved(FROM) string<delimiter(`)content(baz)delimiter(`)> reserved(WHERE) string<delimiter(`)content(bal)delimiter(`)> operator(=) string<delimiter(`)content(quiche)delimiter(`)>operator(;)  comment(-- valid)
diff --git a/test/scanners/sql/selects.expected.raydebug b/test/scanners/sql/selects.expected.raydebug
index 13a85ce..ef1f051 100644
--- a/test/scanners/sql/selects.expected.raydebug
+++ b/test/scanners/sql/selects.expected.raydebug
@@ -14,23 +14,23 @@ comment(--GNU General Public License for more details.)
 comment(--You should have received a copy of the GNU General Public License)
 comment(--along with this program.  If not, see <http://www.gnu.org/licenses/>.)
 
-reserved(SELECT) operator(*) reserved(FROM) ident(users)operator(;)
+class(SELECT) operator(*) reserved(FROM) ident(users)operator(;)
 
-reserved(select) operator(*) reserved(from) ident(users)operator(;)
+class(select) operator(*) reserved(from) ident(users)operator(;)
 
-reserved(SELECT) string<delimiter(`)content(First Name)delimiter(`)>operator(,) string<delimiter(`)content(Last Name)delimiter(`)> reserved(FROM) string<delimiter(`)content(User Table)delimiter(`)>operator(;)
+class(SELECT) string<delimiter(`)content(First Name)delimiter(`)>operator(,) string<delimiter(`)content(Last Name)delimiter(`)> reserved(FROM) string<delimiter(`)content(User Table)delimiter(`)>operator(;)
 
-reserved(select) ident(first_name)operator(,) ident(last_name) reserved(FROM) ident(users)operator(;)
+class(select) ident(first_name)operator(,) ident(last_name) reserved(FROM) ident(users)operator(;)
 
-reserved(select) ident(first_name) operator(|)operator(|) string<delimiter(')content( )delimiter(')> operator(|)operator(|) ident(last_name) reserved(from) ident(users)operator(;)
+class(select) ident(first_name) operator(|)operator(|) string<delimiter(')content( )delimiter(')> operator(|)operator(|) ident(last_name) reserved(from) ident(users)operator(;)
 
-reserved(select) ident(first_name) operator(|)operator(|) string<delimiter(")content( )delimiter(")> operator(|)operator(|) ident(last_name) reserved(from) ident(users)operator(;)
+class(select) ident(first_name) operator(|)operator(|) string<delimiter(")content( )delimiter(")> operator(|)operator(|) ident(last_name) reserved(from) ident(users)operator(;)
 
-reserved(SELECT) operator(*) reserved(FROM) ident(users) reserved(JOIN) ident(companies) reserved(USING) operator(()ident(company_id)operator(\)) reserved(WHERE) ident(company_type) operator(=) string<delimiter(')content(Fortune 500)delimiter(')>operator(;)
+class(SELECT) operator(*) reserved(FROM) ident(users) reserved(JOIN) ident(companies) reserved(USING) operator(()ident(company_id)operator(\)) reserved(WHERE) ident(company_type) operator(=) string<delimiter(')content(Fortune 500)delimiter(')>operator(;)
 
-reserved(SELECT) operator(*) reserved(FROM) ident(users) reserved(WHERE) ident(name_first) reserved(LIKE) string<delimiter(')content(%Keith%)delimiter(')>operator(;)
+class(SELECT) operator(*) reserved(FROM) ident(users) reserved(WHERE) ident(name_first) reserved(LIKE) string<delimiter(')content(%Keith%)delimiter(')>operator(;)
 
-reserved(SELECT) reserved(CASE) reserved(WHEN) ident(foo)operator(.)ident(bar) operator(=) string<delimiter(')content(PY)delimiter(')>
+class(SELECT) reserved(CASE) reserved(WHEN) ident(foo)operator(.)ident(bar) operator(=) string<delimiter(')content(PY)delimiter(')>
    reserved(THEN) string<delimiter(')content(BAR)delimiter(')>
    reserved(ELSE) string<delimiter(')content(FOO)delimiter(')>
    reserved(END) reserved(as) ident(bar_type)operator(,)
@@ -42,5 +42,5 @@ reserved(SELECT) reserved(CASE) reserved(WHEN) ident(foo)operator(.)ident(bar) o
    reserved(end)
    operator(\)) ident(over) operator(()reserved(order) reserved(by) ident(id)operator(,) ident(amt)operator(\)) reserved(as) ident(balance)operator(;)
 
-reserved(SELECT) ident(users)operator(.)operator(*)operator(,) operator(()reserved(SELECT) ident(company_name) reserved(FROM) ident(companies) reserved(WHERE) ident(company_id) operator(=) ident(users)operator(.)ident(company_id)operator(\)) reserved(FROM) ident(users)operator(;)
+class(SELECT) ident(users)operator(.)operator(*)operator(,) operator(()class(SELECT) ident(company_name) reserved(FROM) ident(companies) reserved(WHERE) ident(company_id) operator(=) ident(users)operator(.)ident(company_id)operator(\)) reserved(FROM) ident(users)operator(;)