Allow re to compile once

Otherwise re will need to recompile everytime a call to
 tokenize is made.

2 files changed, 11 insertions, 12 deletions

diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index 41edbff..0fd1f31 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -5,6 +5,8 @@
 # This module is part of python-sqlparse and is released under
 # the BSD License: http://www.opensource.org/licenses/bsd-license.php
 
+import re
+
 from sqlparse import tokens
 
 
@@ -68,6 +70,9 @@ SQL_REGEX = {
         (r'[+/@#%^&|`?^-]+', tokens.Operator),
     ]}
 
+FLAGS = re.IGNORECASE | re.UNICODE
+SQL_REGEX = [(re.compile(rx, FLAGS).match, tt) for rx, tt in SQL_REGEX['root']]
+
 KEYWORDS = {
     'ABORT': tokens.Keyword,
     'ABS': tokens.Keyword,
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 781da8a..dd15212 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -12,8 +12,6 @@
 # It's separated from the rest of pygments to increase performance
 # and to allow some customizations.
 
-import re
-
 from sqlparse import tokens
 from sqlparse.keywords import SQL_REGEX
 from sqlparse.compat import StringIO, string_types, text_type
@@ -21,16 +19,12 @@ from sqlparse.utils import consume
 
 
 class Lexer(object):
-    flags = re.IGNORECASE | re.UNICODE
-
-    def __init__(self):
-        self._tokens = []
-
-        for tdef in SQL_REGEX['root']:
-            rex = re.compile(tdef[0], self.flags).match
-            self._tokens.append((rex, tdef[1]))
+    """Lexer
+    Empty class. Leaving for back-support
+    """
 
-    def get_tokens(self, text, encoding=None):
+    @staticmethod
+    def get_tokens(text, encoding=None):
         """
         Return an iterable of (tokentype, value) pairs generated from
         `text`. If `unfiltered` is set to `True`, the filtering mechanism
@@ -57,7 +51,7 @@ class Lexer(object):
 
         iterable = enumerate(text)
         for pos, char in iterable:
-            for rexmatch, action in self._tokens:
+            for rexmatch, action in SQL_REGEX:
                 m = rexmatch(text, pos)
 
                 if not m: