Parse double dollars (PostgreSQL) as literal strings (fixes #277).

2 files changed, 16 insertions, 4 deletions

diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index a6ee1d6..eef0d02 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -10,13 +10,22 @@ import re
 from sqlparse import tokens
 
 
-def is_keyword(value):
+def is_keyword(value, remaining):
     val = value.upper()
     return (KEYWORDS_COMMON.get(val) or
             KEYWORDS_ORACLE.get(val) or
             KEYWORDS.get(val, tokens.Name)), value
 
 
+def parse_literal_string(value, remaining):
+    try:
+        end = remaining[len(value):].index(value)
+    except ValueError:
+        return tokens.Name.Builtin, value
+    literal = remaining[:end + (len(value) * 2)]
+    return tokens.Literal, literal
+
+
 SQL_REGEX = {
     'root': [
         (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint),
@@ -35,7 +44,7 @@ SQL_REGEX = {
 
         (r"`(``|[^`])*`", tokens.Name),
         (r"´(´´|[^´])*´", tokens.Name),
-        (r'\$([_A-Z]\w*)?\$', tokens.Name.Builtin),
+        (r'\$([_A-Z]\w*)?\$', parse_literal_string),
 
         (r'\?', tokens.Name.Placeholder),
         (r'%(\(\w+\))?s', tokens.Name.Placeholder),
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index e7996b2..1979550 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -50,11 +50,14 @@ class Lexer(object):
                 if not m:
                     continue
                 elif isinstance(action, tokens._TokenType):
+                    consume_pos = m.end() - pos - 1
                     yield action, m.group()
                 elif callable(action):
-                    yield action(m.group())
+                    ttype, value = action(m.group(), text[pos:])
+                    consume_pos = len(value) - 1
+                    yield ttype, value
 
-                consume(iterable, m.end() - pos - 1)
+                consume(iterable, consume_pos)
                 break
             else:
                 yield tokens.Error, char