From 348ff620fa1acb807b83b173ee62807df21510e5 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 09:55:41 -0700 Subject: Simplify multi-line comments --- sqlparse/keywords.py | 11 +++-------- sqlparse/lexer.py | 34 ++++++---------------------------- 2 files changed, 9 insertions(+), 36 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index c6db0a9..41edbff 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -19,9 +19,11 @@ SQL_REGEX = { # $ matches *before* newline, therefore we have two patterns # to match Comment.Single (r'(--|# ).*?$', tokens.Comment.Single), + (r'/\*[\s\S]*?\*/', tokens.Comment.Multiline), + (r'(\r\n|\r|\n)', tokens.Newline), (r'\s+', tokens.Whitespace), - (r'/\*', tokens.Comment.Multiline, 'multiline-comments'), + (r':=', tokens.Assignment), (r'::', tokens.Punctuation), (r'[*]', tokens.Wildcard), @@ -64,12 +66,6 @@ SQL_REGEX = { (r'[;:()\[\],\.]', tokens.Punctuation), (r'[<>=~!]+', tokens.Operator.Comparison), (r'[+/@#%^&|`?^-]+', tokens.Operator), - ], - 'multiline-comments': [ - (r'/\*', tokens.Comment.Multiline, 'multiline-comments'), - (r'\*/', tokens.Comment.Multiline, '#pop'), - (r'[^/\*]+', tokens.Comment.Multiline), - (r'[/*]', tokens.Comment.Multiline), ]} KEYWORDS = { @@ -600,7 +596,6 @@ KEYWORDS = { 'VARYING': tokens.Name.Builtin, } - KEYWORDS_COMMON = { 'SELECT': tokens.Keyword.DML, 'INSERT': tokens.Keyword.DML, diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 84c8e78..781da8a 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -24,21 +24,11 @@ class Lexer(object): flags = re.IGNORECASE | re.UNICODE def __init__(self): - self._tokens = {} - - for state in SQL_REGEX: - self._tokens[state] = [] - - for tdef in SQL_REGEX[state]: - rex = re.compile(tdef[0], self.flags).match - new_state = None - if len(tdef) > 2: - # Only Multiline comments - if tdef[2] == '#pop': - new_state = -1 - elif tdef[2] in SQL_REGEX: - new_state = (tdef[2],) - self._tokens[state].append((rex, tdef[1], new_state)) + self._tokens = [] + + for tdef in SQL_REGEX['root']: + rex = re.compile(tdef[0], self.flags).match + self._tokens.append((rex, tdef[1])) def get_tokens(self, text, encoding=None): """ @@ -54,8 +44,6 @@ class Lexer(object): ``stack`` is the inital stack (default: ``['root']``) """ encoding = encoding or 'utf-8' - statestack = ['root', ] - statetokens = self._tokens['root'] if isinstance(text, string_types): text = StringIO(text) @@ -69,7 +57,7 @@ class Lexer(object): iterable = enumerate(text) for pos, char in iterable: - for rexmatch, action, new_state in statetokens: + for rexmatch, action in self._tokens: m = rexmatch(text, pos) if not m: @@ -79,16 +67,6 @@ class Lexer(object): elif callable(action): yield action(m.group()) - if isinstance(new_state, tuple): - for state in new_state: - # fixme: multiline-comments not stackable - if not (state == 'multiline-comments' - and statestack[-1] == 'multiline-comments'): - statestack.append(state) - elif isinstance(new_state, int): - del statestack[new_state:] - statetokens = self._tokens[statestack[-1]] - consume(iterable, m.end() - pos - 1) break else: -- cgit v1.2.1 From 41cbd6a86d1550b6e1634bc0f3b203dabcc2698f Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 10:34:12 -0700 Subject: Allow re to compile once Otherwise re will need to recompile everytime a call to tokenize is made. --- sqlparse/keywords.py | 5 +++++ sqlparse/lexer.py | 18 ++++++------------ 2 files changed, 11 insertions(+), 12 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index 41edbff..0fd1f31 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -5,6 +5,8 @@ # This module is part of python-sqlparse and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +import re + from sqlparse import tokens @@ -68,6 +70,9 @@ SQL_REGEX = { (r'[+/@#%^&|`?^-]+', tokens.Operator), ]} +FLAGS = re.IGNORECASE | re.UNICODE +SQL_REGEX = [(re.compile(rx, FLAGS).match, tt) for rx, tt in SQL_REGEX['root']] + KEYWORDS = { 'ABORT': tokens.Keyword, 'ABS': tokens.Keyword, diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 781da8a..dd15212 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -12,8 +12,6 @@ # It's separated from the rest of pygments to increase performance # and to allow some customizations. -import re - from sqlparse import tokens from sqlparse.keywords import SQL_REGEX from sqlparse.compat import StringIO, string_types, text_type @@ -21,16 +19,12 @@ from sqlparse.utils import consume class Lexer(object): - flags = re.IGNORECASE | re.UNICODE - - def __init__(self): - self._tokens = [] - - for tdef in SQL_REGEX['root']: - rex = re.compile(tdef[0], self.flags).match - self._tokens.append((rex, tdef[1])) + """Lexer + Empty class. Leaving for back-support + """ - def get_tokens(self, text, encoding=None): + @staticmethod + def get_tokens(text, encoding=None): """ Return an iterable of (tokentype, value) pairs generated from `text`. If `unfiltered` is set to `True`, the filtering mechanism @@ -57,7 +51,7 @@ class Lexer(object): iterable = enumerate(text) for pos, char in iterable: - for rexmatch, action in self._tokens: + for rexmatch, action in SQL_REGEX: m = rexmatch(text, pos) if not m: -- cgit v1.2.1 From 5d0810b5ce607a5a5a92ce065424a17a924886a7 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 11:22:26 -0700 Subject: Improve Comment Regex and add Comment.Hint for Oracle --- sqlparse/keywords.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index 0fd1f31..a14fa55 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -17,10 +17,10 @@ def is_keyword(value): SQL_REGEX = { 'root': [ - (r'(--|# ).*?(\r\n|\r|\n)', tokens.Comment.Single), - # $ matches *before* newline, therefore we have two patterns - # to match Comment.Single - (r'(--|# ).*?$', tokens.Comment.Single), + (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint), + (r'/\*\+[\s\S]*?\*/', tokens.Comment.Multiline.Hint), + + (r'(--|# ).*?(\r\n|\r|\n|$)', tokens.Comment.Single), (r'/\*[\s\S]*?\*/', tokens.Comment.Multiline), (r'(\r\n|\r|\n)', tokens.Newline), -- cgit v1.2.1 From 2fcbee71724dd6311afe5a88106c4591bc1914ee Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 11:24:56 -0700 Subject: Rewrite wildcard and placeholder regex --- sqlparse/keywords.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index a14fa55..9c4495b 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -28,15 +28,18 @@ SQL_REGEX = { (r':=', tokens.Assignment), (r'::', tokens.Punctuation), - (r'[*]', tokens.Wildcard), + + (r'\*', tokens.Wildcard), + (r'CASE\b', tokens.Keyword), # extended CASE(foo) (r"`(``|[^`])*`", tokens.Name), (r"´(´´|[^´])*´", tokens.Name), (r'\$([^\W\d]\w*)?\$', tokens.Name.Builtin), - (r'\?{1}', tokens.Name.Placeholder), - (r'%\(\w+\)s', tokens.Name.Placeholder), - (r'%s', tokens.Name.Placeholder), + + (r'\?', tokens.Name.Placeholder), + (r'%(\(\w+\))?s', tokens.Name.Placeholder), (r'[$:?]\w+', tokens.Name.Placeholder), + # FIXME(andi): VALUES shouldn't be listed here # see https://github.com/andialbrecht/sqlparse/pull/64 (r'VALUES', tokens.Keyword), -- cgit v1.2.1 From cca348e295fc2e00c745d3a8c1bac3e8d4a5d97e Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 11:32:15 -0700 Subject: Refactor regex Func/keyword exceptions together --- sqlparse/keywords.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index 9c4495b..fb2f042 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -31,7 +31,6 @@ SQL_REGEX = { (r'\*', tokens.Wildcard), - (r'CASE\b', tokens.Keyword), # extended CASE(foo) (r"`(``|[^`])*`", tokens.Name), (r"´(´´|[^´])*´", tokens.Name), (r'\$([^\W\d]\w*)?\$', tokens.Name.Builtin), @@ -42,12 +41,11 @@ SQL_REGEX = { # FIXME(andi): VALUES shouldn't be listed here # see https://github.com/andialbrecht/sqlparse/pull/64 - (r'VALUES', tokens.Keyword), - (r'(@|##|#)[^\W\d_]\w+', tokens.Name), # IN is special, it may be followed by a parenthesis, but # is never a functino, see issue183 - (r'in\b(?=[ (])?', tokens.Keyword), - (r'USING(?=\()', tokens.Keyword), + (r'(CASE|IN|VALUES|USING)\b', tokens.Keyword), + + (r'(@|##|#)[^\W\d_]\w+', tokens.Name), (r'[^\W\d_]\w*(?=[.(])', tokens.Name), # see issue39 (r'[-]?0x[0-9a-fA-F]+', tokens.Number.Hexadecimal), (r'[-]?[0-9]*(\.[0-9]+)?[eE][-]?[0-9]+', tokens.Number.Float), -- cgit v1.2.1 From 752100443bd1a5ddd81d57166db887c1fbd9f7ef Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 11:36:07 -0700 Subject: Simplify regex [^\W\d_] [^\W\d_] = [a-z] (when ignoring case) not( not (a-z, numbers, or _) or number or _ ) [(a-z or number or _) and not number and not _] [A-Z] Correct not null spacing --- sqlparse/keywords.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index fb2f042..1b5d5f2 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -33,7 +33,7 @@ SQL_REGEX = { (r"`(``|[^`])*`", tokens.Name), (r"´(´´|[^´])*´", tokens.Name), - (r'\$([^\W\d]\w*)?\$', tokens.Name.Builtin), + (r'\$([_A-Z]\w*)?\$', tokens.Name.Builtin), (r'\?', tokens.Name.Placeholder), (r'%(\(\w+\))?s', tokens.Name.Placeholder), @@ -45,8 +45,11 @@ SQL_REGEX = { # is never a functino, see issue183 (r'(CASE|IN|VALUES|USING)\b', tokens.Keyword), - (r'(@|##|#)[^\W\d_]\w+', tokens.Name), - (r'[^\W\d_]\w*(?=[.(])', tokens.Name), # see issue39 + (r'(@|##|#)[A-Z]\w+', tokens.Name), + (r'[A-Z]\w*(?=\.)', tokens.Name), # see issue39 + (r'(?<=\.)[A-Z]\w*', tokens.Name), # .'Name' + (r'[A-Z]\w*(?=\()', tokens.Name), # side effect: change kw to func + (r'[-]?0x[0-9a-fA-F]+', tokens.Number.Hexadecimal), (r'[-]?[0-9]*(\.[0-9]+)?[eE][-]?[0-9]+', tokens.Number.Float), (r'[-]?[0-9]*\.[0-9]+', tokens.Number.Float), @@ -61,11 +64,12 @@ SQL_REGEX = { (r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?' r'|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword), (r'END(\s+IF|\s+LOOP|\s+WHILE)?\b', tokens.Keyword), - (r'NOT NULL\b', tokens.Keyword), + (r'NOT\s+NULL\b', tokens.Keyword), (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL), (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin), - (r'(?<=\.)[^\W\d_]\w*', tokens.Name), - (r'[^\W\d]\w*', is_keyword), + + (r'[_A-Z]\w*', is_keyword), + (r'[;:()\[\],\.]', tokens.Punctuation), (r'[<>=~!]+', tokens.Operator.Comparison), (r'[+/@#%^&|`?^-]+', tokens.Operator), -- cgit v1.2.1 From 689153a1fbe1bd8661c405e5305c4ac2414e55a8 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 11:37:12 -0700 Subject: Clean regex for Number tokens --- sqlparse/keywords.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index 1b5d5f2..34e4d80 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -50,10 +50,12 @@ SQL_REGEX = { (r'(?<=\.)[A-Z]\w*', tokens.Name), # .'Name' (r'[A-Z]\w*(?=\()', tokens.Name), # side effect: change kw to func - (r'[-]?0x[0-9a-fA-F]+', tokens.Number.Hexadecimal), - (r'[-]?[0-9]*(\.[0-9]+)?[eE][-]?[0-9]+', tokens.Number.Float), - (r'[-]?[0-9]*\.[0-9]+', tokens.Number.Float), - (r'[-]?[0-9]+', tokens.Number.Integer), + # TODO: `1.` and `.1` are valid numbers + (r'-?0x[\dA-F]+', tokens.Number.Hexadecimal), + (r'-?\d*(\.\d+)?E-?\d+', tokens.Number.Float), + (r'-?\d*\.\d+', tokens.Number.Float), + (r'-?\d+', tokens.Number.Integer), + (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single), # not a real string literal in ANSI SQL: (r'(""|".*?[^\\]")', tokens.String.Symbol), -- cgit v1.2.1 From ece7c26727b28ea05feeeba3a9c2d1b4b7eb5c54 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Fri, 3 Jun 2016 11:39:29 -0700 Subject: Rewrite regex to allow spaces between `name` and `.` --- sqlparse/keywords.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'sqlparse') diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index 34e4d80..38b7765 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -46,7 +46,11 @@ SQL_REGEX = { (r'(CASE|IN|VALUES|USING)\b', tokens.Keyword), (r'(@|##|#)[A-Z]\w+', tokens.Name), - (r'[A-Z]\w*(?=\.)', tokens.Name), # see issue39 + + # see issue #39 + # Spaces around period `schema . name` are valid identifier + # TODO: Spaces before period not implemented + (r'[A-Z]\w*(?=\s*\.)', tokens.Name), # 'Name' . (r'(?<=\.)[A-Z]\w*', tokens.Name), # .'Name' (r'[A-Z]\w*(?=\()', tokens.Name), # side effect: change kw to func -- cgit v1.2.1