From daf207cad41e1f04a16fd3d92ab41685b0fe4543 Mon Sep 17 00:00:00 2001 From: quest Date: Sun, 22 Apr 2012 14:48:38 +0200 Subject: VALUES was incorrectly treated as a function --- sqlparse/lexer.py | 1 + 1 file changed, 1 insertion(+) (limited to 'sqlparse') diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index dc794ab..c10e136 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -179,6 +179,7 @@ class Lexer(object): (r'\$([a-zA-Z_][a-zA-Z0-9_]*)?\$', tokens.Name.Builtin), (r'\?{1}', tokens.Name.Placeholder), (r'[$:?%][a-zA-Z0-9_]+', tokens.Name.Placeholder), + (r'VALUES', tokens.Keyword), (r'@[a-zA-Z_][a-zA-Z0-9_]+', tokens.Name), (r'[a-zA-Z_][a-zA-Z0-9_]*(?=[.(])', tokens.Name), # see issue39 (r'[-]?0x[0-9a-fA-F]+', tokens.Number.Hexadecimal), -- cgit v1.2.1 From 5cc2604dea7d6004ec9607fe77b48abf27d8e3ee Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Mon, 23 Apr 2012 09:30:44 +0200 Subject: Add FIXME. --- sqlparse/lexer.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'sqlparse') diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index c10e136..5b0f116 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -179,6 +179,8 @@ class Lexer(object): (r'\$([a-zA-Z_][a-zA-Z0-9_]*)?\$', tokens.Name.Builtin), (r'\?{1}', tokens.Name.Placeholder), (r'[$:?%][a-zA-Z0-9_]+', tokens.Name.Placeholder), + # FIXME(andi): VALUES shouldn't be listed here + # see https://github.com/andialbrecht/sqlparse/pull/64 (r'VALUES', tokens.Keyword), (r'@[a-zA-Z_][a-zA-Z0-9_]+', tokens.Name), (r'[a-zA-Z_][a-zA-Z0-9_]*(?=[.(])', tokens.Name), # see issue39 -- cgit v1.2.1 From 7f422d3e99bb21801db6f65619c876f709cdf44a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 17 May 2012 16:38:10 +0200 Subject: Optimized IncludeStatement --- sqlparse/filters.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 291a613..9ee2ce0 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -110,17 +110,15 @@ class IncludeStatement: elif self.detected: # Omit whitespaces if token_type in Whitespace: - pass - - # Get path of file to include - path = None + continue + # Found file path to include if token_type in String.Symbol: # if token_type in tokens.String.Symbol: + + # Get path of file to include path = join(self.dirpath, value[1:-1]) - # Include file if path was found - if path: try: f = open(path) raw_sql = f.read() -- cgit v1.2.1 From 93eb88cbc88d239e70268b98ab9a96236c73efee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 17 May 2012 16:44:52 +0200 Subject: Raise an exception on max recursion limit reached --- sqlparse/filters.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 9ee2ce0..8430687 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -93,6 +93,9 @@ class IncludeStatement: """Filter that enable a INCLUDE statement""" def __init__(self, dirpath=".", maxRecursive=10): + if maxRecursive <= 0: + raise ValueError('Max recursion limit reached') + self.dirpath = abspath(dirpath) self.maxRecursive = maxRecursive @@ -131,7 +134,8 @@ class IncludeStatement: # and add all its tokens to the main stack recursively # [ToDo] Add maximum recursive iteration value stack = FilterStack() - stack.preprocess.append(IncludeStatement(self.dirpath)) + stack.preprocess.append(IncludeStatement(self.dirpath, + self.maxRecursive - 1)) for tv in stack.run(raw_sql): yield tv -- cgit v1.2.1 From f141b2d344bbd09a7469d5c9ecffb38ed6ccd618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 17 May 2012 16:52:01 +0200 Subject: Raise exception on INCLUDE loading failure --- sqlparse/filters.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 8430687..e17766d 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -92,12 +92,13 @@ def StripWhitespace(stream): class IncludeStatement: """Filter that enable a INCLUDE statement""" - def __init__(self, dirpath=".", maxRecursive=10): + def __init__(self, dirpath=".", maxRecursive=10, raiseonload=False): if maxRecursive <= 0: raise ValueError('Max recursion limit reached') self.dirpath = abspath(dirpath) self.maxRecursive = maxRecursive + self.raiseonload = raiseonload self.detected = False @@ -127,15 +128,22 @@ class IncludeStatement: raw_sql = f.read() f.close() except IOError, err: + # Raise the exception to the interpreter + if self.raiseonload: + raise + + # Put the exception as a comment on the SQL code yield Comment, u'-- IOError: %s\n' % err else: # Create new FilterStack to parse readed file # and add all its tokens to the main stack recursively - # [ToDo] Add maximum recursive iteration value + filtr = IncludeStatement(self.dirpath, + self.maxRecursive - 1, + self.raiseonload) + stack = FilterStack() - stack.preprocess.append(IncludeStatement(self.dirpath, - self.maxRecursive - 1)) + stack.preprocess.append(filtr) for tv in stack.run(raw_sql): yield tv -- cgit v1.2.1 From 0aba698a5275f9c437f3142af1c29b5aab387de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 17 May 2012 17:01:47 +0200 Subject: Option to embeb exception on recursion limit --- sqlparse/filters.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index e17766d..17dc960 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -92,13 +92,13 @@ def StripWhitespace(stream): class IncludeStatement: """Filter that enable a INCLUDE statement""" - def __init__(self, dirpath=".", maxRecursive=10, raiseonload=False): + def __init__(self, dirpath=".", maxRecursive=10, raiseexceptions=False): if maxRecursive <= 0: raise ValueError('Max recursion limit reached') self.dirpath = abspath(dirpath) self.maxRecursive = maxRecursive - self.raiseonload = raiseonload + self.raiseexceptions = raiseexceptions self.detected = False @@ -127,9 +127,11 @@ class IncludeStatement: f = open(path) raw_sql = f.read() f.close() + + # There was a problem loading the include file except IOError, err: # Raise the exception to the interpreter - if self.raiseonload: + if self.raiseexceptions: raise # Put the exception as a comment on the SQL code @@ -138,9 +140,19 @@ class IncludeStatement: else: # Create new FilterStack to parse readed file # and add all its tokens to the main stack recursively - filtr = IncludeStatement(self.dirpath, - self.maxRecursive - 1, - self.raiseonload) + try: + filtr = IncludeStatement(self.dirpath, + self.maxRecursive - 1, + self.raiseexceptions) + + # Max recursion limit reached + except ValueError, err: + # Raise the exception to the interpreter + if self.raiseexceptions: + raise + + # Put the exception as a comment on the SQL code + yield Comment, u'-- ValueError: %s\n' % err stack = FilterStack() stack.preprocess.append(filtr) -- cgit v1.2.1 From 8b3e88e60d8773ef2432bdc511ed7eaf1dc09a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 17 May 2012 22:34:54 +0200 Subject: Added memoize_generator and use it on IncludeStatement --- sqlparse/filters.py | 9 +++++---- sqlparse/utils.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 sqlparse/utils.py (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 17dc960..92d7bd5 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -4,11 +4,11 @@ import re from os.path import abspath, join -from sqlparse import sql -from sqlparse import tokens as T +from sqlparse import sql, tokens as T from sqlparse.engine import FilterStack -from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation, - String, Whitespace) +from sqlparse.tokens import Comment, Comparison, Keyword, Name, Punctuation +from sqlparse.tokens import String, Whitespace +from sqlparse.utils import memoize_generator # -------------------------- @@ -102,6 +102,7 @@ class IncludeStatement: self.detected = False + @memoize_generator def process(self, stack, stream): # Run over all tokens in the stream for token_type, value in stream: diff --git a/sqlparse/utils.py b/sqlparse/utils.py new file mode 100644 index 0000000..0dbb09f --- /dev/null +++ b/sqlparse/utils.py @@ -0,0 +1,41 @@ +''' +Created on 17/05/2012 + +@author: piranna +''' + + +def memoize_generator(func): + """ + Memoize decorator for generators + + Store `func` results in a cache according to their arguments as 'memoize' + does but instead this works on decorators instead of regular functions. + Obviusly, this is only useful if the generator will always return the same + values for each specific parameters... + """ + cache = {} + + def wrapped_func(*args, **kwargs): + params = (args, kwargs) + + # Look if cached + try: + cached = cache[params] + + # Not cached, exec and store it + except KeyError: + cached = [] + + for item in func(*args, **kwargs): + cached.append(item) + yield item + + cache[params] = cached + + # Cached, yield its items + else: + for item in cached: + yield item + + return wrapped_func -- cgit v1.2.1 From ec110d64b0a5e9f362e76b9c01ecccbd6932d0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 19:20:35 +0200 Subject: Fixed comments by Andi --- sqlparse/filters.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 92d7bd5..08c1f69 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -4,11 +4,11 @@ import re from os.path import abspath, join -from sqlparse import sql, tokens as T +from sqlparse import sql, tokens as T from sqlparse.engine import FilterStack -from sqlparse.tokens import Comment, Comparison, Keyword, Name, Punctuation -from sqlparse.tokens import String, Whitespace -from sqlparse.utils import memoize_generator +from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation, + String, Whitespace) +from sqlparse.utils import memoize_generator # -------------------------- @@ -92,12 +92,12 @@ def StripWhitespace(stream): class IncludeStatement: """Filter that enable a INCLUDE statement""" - def __init__(self, dirpath=".", maxRecursive=10, raiseexceptions=False): - if maxRecursive <= 0: + def __init__(self, dirpath=".", maxrecursive=10, raiseexceptions=False): + if maxrecursive <= 0: raise ValueError('Max recursion limit reached') self.dirpath = abspath(dirpath) - self.maxRecursive = maxRecursive + self.maxRecursive = maxrecursive self.raiseexceptions = raiseexceptions self.detected = False -- cgit v1.2.1 From 7e532bcd9af6f36280f497346d50b4cdd028cfbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 19:21:00 +0200 Subject: Added limit to cache --- sqlparse/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/utils.py b/sqlparse/utils.py index 0dbb09f..fd6651a 100644 --- a/sqlparse/utils.py +++ b/sqlparse/utils.py @@ -6,8 +6,7 @@ Created on 17/05/2012 def memoize_generator(func): - """ - Memoize decorator for generators + """Memoize decorator for generators Store `func` results in a cache according to their arguments as 'memoize' does but instead this works on decorators instead of regular functions. @@ -25,6 +24,12 @@ def memoize_generator(func): # Not cached, exec and store it except KeyError: + # Reset the cache if we have too much cached entries and start over + # In the future would be better to use an OrderedDict and drop the + # Least Recent Used entries + if len(cache) >= 10: + cache.clear() + cached = [] for item in func(*args, **kwargs): -- cgit v1.2.1