Merge pull request #282 from vmuriart/alt_277

Alternate solution for #277

2 files changed, 4 insertions, 16 deletions

diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index eef0d02..dee4030 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -10,22 +10,13 @@ import re
 from sqlparse import tokens
 
 
-def is_keyword(value, remaining):
+def is_keyword(value):
     val = value.upper()
     return (KEYWORDS_COMMON.get(val) or
             KEYWORDS_ORACLE.get(val) or
             KEYWORDS.get(val, tokens.Name)), value
 
 
-def parse_literal_string(value, remaining):
-    try:
-        end = remaining[len(value):].index(value)
-    except ValueError:
-        return tokens.Name.Builtin, value
-    literal = remaining[:end + (len(value) * 2)]
-    return tokens.Literal, literal
-
-
 SQL_REGEX = {
     'root': [
         (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint),
@@ -44,7 +35,7 @@ SQL_REGEX = {
 
         (r"`(``|[^`])*`", tokens.Name),
         (r"´(´´|[^´])*´", tokens.Name),
-        (r'\$([_A-Z]\w*)?\$', parse_literal_string),
+        (r'(\$(?:[_A-Z]\w*)?\$)[\s\S]*?\1', tokens.Literal),
 
         (r'\?', tokens.Name.Placeholder),
         (r'%(\(\w+\))?s', tokens.Name.Placeholder),
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 1979550..e7996b2 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -50,14 +50,11 @@ class Lexer(object):
                 if not m:
                     continue
                 elif isinstance(action, tokens._TokenType):
-                    consume_pos = m.end() - pos - 1
                     yield action, m.group()
                 elif callable(action):
-                    ttype, value = action(m.group(), text[pos:])
-                    consume_pos = len(value) - 1
-                    yield ttype, value
+                    yield action(m.group())
 
-                consume(iterable, consume_pos)
+                consume(iterable, m.end() - pos - 1)
                 break
             else:
                 yield tokens.Error, char