From 83ca5c8ed2134a0f1a1132159c8571060d9b6e91 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 13 Aug 2016 09:21:04 -0700
Subject: Revert "Parse double dollars (PostgreSQL) as literal strings (fixes
 #277)."

This reverts commit 2893bd1857d685cf892beac3a7429d03cf1a09f1.
---
 sqlparse/keywords.py | 13 ++-----------
 sqlparse/lexer.py    |  7 ++-----
 2 files changed, 4 insertions(+), 16 deletions(-)

(limited to 'sqlparse')

diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index eef0d02..a6ee1d6 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -10,22 +10,13 @@ import re
 from sqlparse import tokens
 
 
-def is_keyword(value, remaining):
+def is_keyword(value):
     val = value.upper()
     return (KEYWORDS_COMMON.get(val) or
             KEYWORDS_ORACLE.get(val) or
             KEYWORDS.get(val, tokens.Name)), value
 
 
-def parse_literal_string(value, remaining):
-    try:
-        end = remaining[len(value):].index(value)
-    except ValueError:
-        return tokens.Name.Builtin, value
-    literal = remaining[:end + (len(value) * 2)]
-    return tokens.Literal, literal
-
-
 SQL_REGEX = {
     'root': [
         (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint),
@@ -44,7 +35,7 @@ SQL_REGEX = {
 
         (r"`(``|[^`])*`", tokens.Name),
         (r"´(´´|[^´])*´", tokens.Name),
-        (r'\$([_A-Z]\w*)?\$', parse_literal_string),
+        (r'\$([_A-Z]\w*)?\$', tokens.Name.Builtin),
 
         (r'\?', tokens.Name.Placeholder),
         (r'%(\(\w+\))?s', tokens.Name.Placeholder),
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 1979550..e7996b2 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -50,14 +50,11 @@ class Lexer(object):
                 if not m:
                     continue
                 elif isinstance(action, tokens._TokenType):
-                    consume_pos = m.end() - pos - 1
                     yield action, m.group()
                 elif callable(action):
-                    ttype, value = action(m.group(), text[pos:])
-                    consume_pos = len(value) - 1
-                    yield ttype, value
+                    yield action(m.group())
 
-                consume(iterable, consume_pos)
+                consume(iterable, m.end() - pos - 1)
                 break
             else:
                 yield tokens.Error, char
-- 
cgit v1.2.1


From d11c96aac41521bf6000ca27cbdf330cc656b926 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 13 Aug 2016 09:21:43 -0700
Subject: Alternate solution for #277

---
 sqlparse/keywords.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'sqlparse')

diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index a6ee1d6..dee4030 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -35,7 +35,7 @@ SQL_REGEX = {
 
         (r"`(``|[^`])*`", tokens.Name),
         (r"´(´´|[^´])*´", tokens.Name),
-        (r'\$([_A-Z]\w*)?\$', tokens.Name.Builtin),
+        (r'(\$(?:[_A-Z]\w*)?\$)[\s\S]*?\1', tokens.Literal),
 
         (r'\?', tokens.Name.Placeholder),
         (r'%(\(\w+\))?s', tokens.Name.Placeholder),
-- 
cgit v1.2.1