From a8b46d3b96ead22263de782b36090f2f1bc7c15d Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Tue, 9 Aug 2011 16:03:29 +0200 Subject: Add link to post on mailing list with comments. --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index e11c65c..c222c4f 100644 --- a/TODO +++ b/TODO @@ -1,3 +1,6 @@ +* See + https://groups.google.com/d/msg/sqlparse/huz9lKXt0Lc/11ybIKPJWbUJ + for some interesting hints and suggestions. * Provide a function to replace tokens. See this thread: https://groups.google.com/d/msg/sqlparse/5xmBL2UKqX4/ZX9z_peve-AJ * Fix bugs on issue tracker. * Document filter stack and processing phases. -- cgit v1.2.1 From 36413e8eba08fcadca60f286122b19e9424d55a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 11 Aug 2011 20:40:17 +0200 Subject: Added Get_Comments, StripComments, IncludeStatement, ColumnsSelect and Limit filters. --- sqlparse/filters.py | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 3 deletions(-) diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 813be99..ce2fb80 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -2,8 +2,12 @@ import re -from sqlparse import tokens as T +from os.path import abspath, join + from sqlparse import sql +from sqlparse import tokens as T +from sqlparse.engine import FilterStack +from sqlparse.tokens import Comment, Keyword, Name, Punctuation, String, Whitespace class Filter(object): @@ -52,6 +56,83 @@ class IdentifierCaseFilter(_CaseFilter): yield ttype, value +class Get_Comments(Filter): + """Get the comments from a stack""" + def process(self, stack, stream): + for token_type, value in stream: + if token_type in Comment: + yield token_type, value + + +class StripComments(Filter): + """Strip the comments from a stack""" + def process(self, stack, stream): + for token_type, value in stream: + if token_type not in Comment: + yield token_type, value + + +class IncludeStatement(Filter): + """Filter that enable a INCLUDE statement""" + + def __init__(self, dirpath=".", maxRecursive=10): + self.dirpath = abspath(dirpath) + self.maxRecursive = maxRecursive + + self.detected = False + + + def process(self, stack, stream): + # Run over all tokens in the stream + for token_type, value in stream: + # INCLUDE statement found, set detected mode + if token_type in Name and value.upper() == 'INCLUDE': + self.detected = True + continue + + # INCLUDE statement was found, parse it + elif self.detected: + # Omit whitespaces + if token_type in Whitespace: + pass + + # Get path of file to include + path = None + + if token_type in String.Symbol: +# if token_type in tokens.String.Symbol: + path = join(self.dirpath, value[1:-1]) + + # Include file if path was found + if path: + try: + with open(path) as f: + sql = f.read() + + except IOError, err: + logging.error(err) + yield Comment, u'-- IOError: %s\n' % err + + else: + # Create new FilterStack to parse readed file + # and add all its tokens to the main stack recursively + # [ToDo] Add maximum recursive iteration value + stack = FilterStack() + stack.preprocess.append(IncludeStatement(self.dirpath)) + + for tv in stack.run(sql): + yield tv + + # Set normal mode + self.detected = False + + # Don't include any token while in detected mode + continue + + # Normal token + yield token_type, value + + # ---------------------- # statement process @@ -150,9 +231,9 @@ class ReindentFilter(Filter): t = tlist.token_next_match(i, T.Keyword, split_words, regex=True) if t and t.value.upper() == 'BETWEEN': - t = _next_token(tlist.token_index(t)+1) + t = _next_token(tlist.token_index(t) + 1) if t and t.value.upper() == 'AND': - t = _next_token(tlist.token_index(t)+1) + t = _next_token(tlist.token_index(t) + 1) return t idx = 0 @@ -316,6 +397,56 @@ class RightMarginFilter(Filter): group.tokens = self._process(stack, group, group.tokens) +class ColumnsSelect(Filter): + """Get the columns names of a SELECT query""" + def process(self, stack, stream): + mode = 0 + oldValue = "" + parenthesis = 0 + + for token_type, value in stream: + # Ignore comments + if token_type in Comment: + continue + + # We have not detected a SELECT statement + if mode == 0: + if token_type in Keyword and value == 'SELECT': + mode = 1 + + # We have detected a SELECT statement + elif mode == 1: + if value == 'FROM': + if oldValue: + yield Name, oldValue + + mode = 3 # Columns have been checked + + elif value == 'AS': + oldValue = "" + mode = 2 + + elif token_type == Punctuation and value == ',' and not parenthesis: + if oldValue: + yield Name, oldValue + oldValue = "" + + elif token_type not in Whitespace: + if value == '(': + parenthesis += 1 + elif value == ')': + parenthesis -= 1 + + oldValue += value + + # We are processing an AS keyword + elif mode == 2: + # We check also for Keywords because a bug in SQLParse + if token_type == Name or token_type == Keyword: + yield Name, value + mode = 1 + + # --------------------------- # postprocess @@ -422,3 +553,24 @@ class OutputPHPFilter(Filter): varname = self.varname stmt.tokens = tuple(self._process(stmt.tokens, varname)) return stmt + + +class Limit(Filter): + """Get the LIMIT of a query. + + If not defined, return -1 (SQL specification for no LIMIT query) + """ + def process(self, stack, stream): + index = 7 + stream = list(stream) + stream.reverse() + + # Run over all tokens in the stream from the end + for token_type, value in stream: + index -= 1 + +# if index and token_type in Keyword: + if index and token_type in Keyword and value == 'LIMIT': + return stream[4 - index][1] + + return -1 \ No newline at end of file -- cgit v1.2.1 From 99af50cf179539f8d82e57ea9a0530adad238a96 Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Fri, 12 Aug 2011 14:34:58 +0200 Subject: Code cleanup. --- sqlparse/filters.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sqlparse/filters.py b/sqlparse/filters.py index ce2fb80..2d715a8 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -7,7 +7,10 @@ from os.path import abspath, join from sqlparse import sql from sqlparse import tokens as T from sqlparse.engine import FilterStack -from sqlparse.tokens import Comment, Keyword, Name, Punctuation, String, Whitespace +from sqlparse.tokens import ( + Comment, Keyword, Name, + Punctuation, String, Whitespace, +) class Filter(object): @@ -56,7 +59,7 @@ class IdentifierCaseFilter(_CaseFilter): yield ttype, value -class Get_Comments(Filter): +class GetComments(Filter): """Get the comments from a stack""" def process(self, stack, stream): for token_type, value in stream: @@ -81,7 +84,6 @@ class IncludeStatement(Filter): self.detected = False - def process(self, stack, stream): # Run over all tokens in the stream for token_type, value in stream: @@ -107,10 +109,9 @@ class IncludeStatement(Filter): if path: try: with open(path) as f: - sql = f.read() + raw_sql = f.read() except IOError, err: - logging.error(err) yield Comment, u'-- IOError: %s\n' % err else: @@ -120,7 +121,7 @@ class IncludeStatement(Filter): stack = FilterStack() stack.preprocess.append(IncludeStatement(self.dirpath)) - for tv in stack.run(sql): + for tv in stack.run(raw_sql): yield tv # Set normal mode @@ -227,6 +228,7 @@ class ReindentFilter(Filter): split_words = ('FROM', 'JOIN$', 'AND', 'OR', 'GROUP', 'ORDER', 'UNION', 'VALUES', 'SET', 'BETWEEN') + def _next_token(i): t = tlist.token_next_match(i, T.Keyword, split_words, regex=True) @@ -426,7 +428,8 @@ class ColumnsSelect(Filter): oldValue = "" mode = 2 - elif token_type == Punctuation and value == ',' and not parenthesis: + elif (token_type == Punctuation + and value == ',' and not parenthesis): if oldValue: yield Name, oldValue oldValue = "" @@ -573,4 +576,4 @@ class Limit(Filter): if index and token_type in Keyword and value == 'LIMIT': return stream[4 - index][1] - return -1 \ No newline at end of file + return -1 -- cgit v1.2.1 From 5030c7060e33f8390a618530f44778277a915416 Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Fri, 12 Aug 2011 14:40:30 +0200 Subject: =?UTF-8?q?Add=20Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20to=20AUTHO?= =?UTF-8?q?RS.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AUTHORS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 4aec012..b12598f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,3 +3,9 @@ python-sqlparse is written and maintained by Andi Albrecht Date: Fri, 12 Aug 2011 14:42:57 +0200 Subject: Remove duplicate "Contributors" line. --- AUTHORS | 2 -- 1 file changed, 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index b12598f..10b68b8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,8 +3,6 @@ python-sqlparse is written and maintained by Andi Albrecht Date: Fri, 12 Aug 2011 14:56:21 +0200 Subject: Remove with statement to stay as compatible with older Python versions as possible. For now, sqlparse is still compatible with Python 2.4 :) --- sqlparse/filters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 2d715a8..cba7b8f 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -108,9 +108,9 @@ class IncludeStatement(Filter): # Include file if path was found if path: try: - with open(path) as f: - raw_sql = f.read() - + f = open(path) + raw_sql = f.read() + f.close() except IOError, err: yield Comment, u'-- IOError: %s\n' % err -- cgit v1.2.1