diff options
Diffstat (limited to 'sqlparse')
| -rw-r--r-- | sqlparse/keywords.py | 13 | ||||
| -rw-r--r-- | sqlparse/lexer.py | 7 |
2 files changed, 16 insertions, 4 deletions
diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index a6ee1d6..eef0d02 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -10,13 +10,22 @@ import re from sqlparse import tokens -def is_keyword(value): +def is_keyword(value, remaining): val = value.upper() return (KEYWORDS_COMMON.get(val) or KEYWORDS_ORACLE.get(val) or KEYWORDS.get(val, tokens.Name)), value +def parse_literal_string(value, remaining): + try: + end = remaining[len(value):].index(value) + except ValueError: + return tokens.Name.Builtin, value + literal = remaining[:end + (len(value) * 2)] + return tokens.Literal, literal + + SQL_REGEX = { 'root': [ (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint), @@ -35,7 +44,7 @@ SQL_REGEX = { (r"`(``|[^`])*`", tokens.Name), (r"´(´´|[^´])*´", tokens.Name), - (r'\$([_A-Z]\w*)?\$', tokens.Name.Builtin), + (r'\$([_A-Z]\w*)?\$', parse_literal_string), (r'\?', tokens.Name.Placeholder), (r'%(\(\w+\))?s', tokens.Name.Placeholder), diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index e7996b2..1979550 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -50,11 +50,14 @@ class Lexer(object): if not m: continue elif isinstance(action, tokens._TokenType): + consume_pos = m.end() - pos - 1 yield action, m.group() elif callable(action): - yield action(m.group()) + ttype, value = action(m.group(), text[pos:]) + consume_pos = len(value) - 1 + yield ttype, value - consume(iterable, m.end() - pos - 1) + consume(iterable, consume_pos) break else: yield tokens.Error, char |