Simplify multi-line comments

1 files changed, 6 insertions, 28 deletions

diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 84c8e78..781da8a 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -24,21 +24,11 @@ class Lexer(object):
     flags = re.IGNORECASE | re.UNICODE
 
     def __init__(self):
-        self._tokens = {}
-
-        for state in SQL_REGEX:
-            self._tokens[state] = []
-
-            for tdef in SQL_REGEX[state]:
-                rex = re.compile(tdef[0], self.flags).match
-                new_state = None
-                if len(tdef) > 2:
-                    # Only Multiline comments
-                    if tdef[2] == '#pop':
-                        new_state = -1
-                    elif tdef[2] in SQL_REGEX:
-                        new_state = (tdef[2],)
-                self._tokens[state].append((rex, tdef[1], new_state))
+        self._tokens = []
+
+        for tdef in SQL_REGEX['root']:
+            rex = re.compile(tdef[0], self.flags).match
+            self._tokens.append((rex, tdef[1]))
 
     def get_tokens(self, text, encoding=None):
         """
@@ -54,8 +44,6 @@ class Lexer(object):
         ``stack`` is the inital stack (default: ``['root']``)
         """
         encoding = encoding or 'utf-8'
-        statestack = ['root', ]
-        statetokens = self._tokens['root']
 
         if isinstance(text, string_types):
             text = StringIO(text)
@@ -69,7 +57,7 @@ class Lexer(object):
 
         iterable = enumerate(text)
         for pos, char in iterable:
-            for rexmatch, action, new_state in statetokens:
+            for rexmatch, action in self._tokens:
                 m = rexmatch(text, pos)
 
                 if not m:
@@ -79,16 +67,6 @@ class Lexer(object):
                 elif callable(action):
                     yield action(m.group())
 
-                if isinstance(new_state, tuple):
-                    for state in new_state:
-                        # fixme: multiline-comments not stackable
-                        if not (state == 'multiline-comments'
-                                and statestack[-1] == 'multiline-comments'):
-                            statestack.append(state)
-                elif isinstance(new_state, int):
-                    del statestack[new_state:]
-                statetokens = self._tokens[statestack[-1]]
-
                 consume(iterable, m.end() - pos - 1)
                 break
             else: