Merge pull request #282 from vmuriart/alt_277

Alternate solution for #277

1 files changed, 2 insertions, 5 deletions

diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 1979550..e7996b2 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -50,14 +50,11 @@ class Lexer(object):
                 if not m:
                     continue
                 elif isinstance(action, tokens._TokenType):
-                    consume_pos = m.end() - pos - 1
                     yield action, m.group()
                 elif callable(action):
-                    ttype, value = action(m.group(), text[pos:])
-                    consume_pos = len(value) - 1
-                    yield ttype, value
+                    yield action(m.group())
 
-                consume(iterable, consume_pos)
+                consume(iterable, m.end() - pos - 1)
                 break
             else:
                 yield tokens.Error, char