From 5ce225522ba2b2a8af23c7efcbd6261bd9f09528 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Thu, 2 Jun 2016 21:25:01 -0700 Subject: Remove as parameter stack in filter.process --- sqlparse/engine/__init__.py | 8 ++++---- sqlparse/engine/filter.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'sqlparse/engine') diff --git a/sqlparse/engine/__init__.py b/sqlparse/engine/__init__.py index 1c2bf09..e69a138 100644 --- a/sqlparse/engine/__init__.py +++ b/sqlparse/engine/__init__.py @@ -29,12 +29,12 @@ class FilterStack(object): # Process token stream if self.preprocess: for filter_ in self.preprocess: - stream = filter_.process(self, stream) + stream = filter_.process(stream) if (self.stmtprocess or self.postprocess or self.split_statements or self._grouping): splitter = StatementFilter() - stream = splitter.process(self, stream) + stream = splitter.process(stream) if self._grouping: @@ -50,7 +50,7 @@ class FilterStack(object): ret = [] for stmt in stream: for filter_ in self.stmtprocess: - filter_.process(self, stmt) + filter_.process(stmt) ret.append(stmt) return ret stream = _run1(stream) @@ -61,7 +61,7 @@ class FilterStack(object): for stmt in stream: stmt.tokens = list(stmt.flatten()) for filter_ in self.postprocess: - stmt = filter_.process(self, stmt) + stmt = filter_.process(stmt) yield stmt stream = _run2(stream) diff --git a/sqlparse/engine/filter.py b/sqlparse/engine/filter.py index 71020e7..3847e3d 100644 --- a/sqlparse/engine/filter.py +++ b/sqlparse/engine/filter.py @@ -76,7 +76,7 @@ class StatementFilter(object): # Default return 0 - def process(self, stack, stream): + def process(self, stream): "Process the stream" consume_ws = False splitlevel = 0 -- cgit v1.2.1 From 3fed0393a80a40ea28e5fc0cea9b526630e9f42b Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Thu, 2 Jun 2016 21:38:23 -0700 Subject: Refactor filter-stack to simplify logic if (self.stmtprocess or self.postprocess or self.split_statements or self._grouping): always evaluates to true after removing unused features --- sqlparse/engine/__init__.py | 48 ++++++++++++--------------------------------- sqlparse/engine/grouping.py | 5 +++-- 2 files changed, 15 insertions(+), 38 deletions(-) (limited to 'sqlparse/engine') diff --git a/sqlparse/engine/__init__.py b/sqlparse/engine/__init__.py index e69a138..7f00c57 100644 --- a/sqlparse/engine/__init__.py +++ b/sqlparse/engine/__init__.py @@ -13,12 +13,10 @@ from sqlparse.engine.filter import StatementFilter class FilterStack(object): - def __init__(self): self.preprocess = [] self.stmtprocess = [] self.postprocess = [] - self.split_statements = False self._grouping = False def enable_grouping(self): @@ -27,42 +25,20 @@ class FilterStack(object): def run(self, sql, encoding=None): stream = lexer.tokenize(sql, encoding) # Process token stream - if self.preprocess: - for filter_ in self.preprocess: - stream = filter_.process(stream) - - if (self.stmtprocess or self.postprocess or - self.split_statements or self._grouping): - splitter = StatementFilter() - stream = splitter.process(stream) - - if self._grouping: - - def _group(stream): - for stmt in stream: - grouping.group(stmt) - yield stmt - stream = _group(stream) + for filter_ in self.preprocess: + stream = filter_.process(stream) - if self.stmtprocess: + stream = StatementFilter().process(stream) - def _run1(stream): - ret = [] - for stmt in stream: - for filter_ in self.stmtprocess: - filter_.process(stmt) - ret.append(stmt) - return ret - stream = _run1(stream) + # Output: Stream processed Statements + for stmt in stream: + if self._grouping: + stmt = grouping.group(stmt) - if self.postprocess: + for filter_ in self.stmtprocess: + filter_.process(stmt) - def _run2(stream): - for stmt in stream: - stmt.tokens = list(stmt.flatten()) - for filter_ in self.postprocess: - stmt = filter_.process(stmt) - yield stmt - stream = _run2(stream) + for filter_ in self.postprocess: + stmt = filter_.process(stmt) - return stream + yield stmt diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 0ac1cb3..c680995 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -266,7 +266,7 @@ def align_comments(tlist): token = tlist.token_next_by(i=sql.Comment, idx=token) -def group(tlist): +def group(stmt): for func in [ group_comments, group_brackets, @@ -291,4 +291,5 @@ def group(tlist): group_foreach, group_begin, ]: - func(tlist) + func(stmt) + return stmt -- cgit v1.2.1 From f0a6af57b7a5c116528db73643b26f934477d350 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Thu, 2 Jun 2016 22:40:37 -0700 Subject: Minor refactor statement filter and add comments --- sqlparse/engine/filter.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) (limited to 'sqlparse/engine') diff --git a/sqlparse/engine/filter.py b/sqlparse/engine/filter.py index 3847e3d..c7b3bf8 100644 --- a/sqlparse/engine/filter.py +++ b/sqlparse/engine/filter.py @@ -10,41 +10,51 @@ from sqlparse import tokens as T class StatementFilter(object): - "Filter that split stream at individual statements" + """Filter that split stream at individual statements""" def __init__(self): - self._in_declare = False - self._in_dbldollar = False - self._is_create = False - self._begin_depth = 0 + self._reset() def _reset(self): - "Set the filter attributes to its default values" + """Set the filter attributes to its default values""" self._in_declare = False self._in_dbldollar = False self._is_create = False self._begin_depth = 0 def _change_splitlevel(self, ttype, value): - "Get the new split level (increase, decrease or remain equal)" + """Get the new split level (increase, decrease or remain equal)""" # PostgreSQL if ttype == T.Name.Builtin \ and value.startswith('$') and value.endswith('$'): + + # 2nd dbldollar found. $quote$ completed + # decrease level if self._in_dbldollar: self._in_dbldollar = False return -1 else: self._in_dbldollar = True return 1 + + # if inside $$ everything inside is defining function character. + # Nothing inside can create a new statement elif self._in_dbldollar: return 0 # ANSI + # if normal token return + # wouldn't parenthesis increase/decrease a level? + # no, inside a paranthesis can't start new statement if ttype not in T.Keyword: return 0 + # Everything after here is ttype = T.Keyword + # Also to note, once entered an If statement you are done and basically + # returning unified = value.upper() + # can have nested declare inside of being... if unified == 'DECLARE' and self._is_create and self._begin_depth == 0: self._in_declare = True return 1 @@ -59,12 +69,16 @@ class StatementFilter(object): if unified in ('END IF', 'END FOR', 'END WHILE'): return -1 + # Should this respect a preceeding BEGIN? + # In CASE ... WHEN ... END this results in a split level -1. + # Would having multiple CASE WHEN END and a Assigment Operator + # cause the statement to cut off prematurely? if unified == 'END': - # Should this respect a preceeding BEGIN? - # In CASE ... WHEN ... END this results in a split level -1. self._begin_depth = max(0, self._begin_depth - 1) return -1 + # three keywords begin with CREATE, but only one of them is DDL + # DDL Create though can contain more words such as "or replace" if ttype is T.Keyword.DDL and unified.startswith('CREATE'): self._is_create = True return 0 @@ -77,7 +91,7 @@ class StatementFilter(object): return 0 def process(self, stream): - "Process the stream" + """Process the stream""" consume_ws = False splitlevel = 0 stmt = None @@ -86,6 +100,9 @@ class StatementFilter(object): # Run over all stream tokens for ttype, value in stream: # Yield token if we finished a statement and there's no whitespaces + # It will count newline token as a non whitespace. In this context + # whitespace ignores newlines. + # why don't multi line comments also count? if consume_ws and ttype not in (T.Whitespace, T.Comment.Single): stmt.tokens = stmt_tokens yield stmt -- cgit v1.2.1 From 2b8ede11388e81e0f6dc871a45c5327eaf456e44 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Thu, 2 Jun 2016 22:54:25 -0700 Subject: Refactor statement filter --- sqlparse/engine/filter.py | 65 ++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 38 deletions(-) (limited to 'sqlparse/engine') diff --git a/sqlparse/engine/filter.py b/sqlparse/engine/filter.py index c7b3bf8..ea2033a 100644 --- a/sqlparse/engine/filter.py +++ b/sqlparse/engine/filter.py @@ -5,8 +5,7 @@ # This module is part of python-sqlparse and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -from sqlparse.sql import Statement, Token -from sqlparse import tokens as T +from sqlparse import sql, tokens as T class StatementFilter(object): @@ -22,11 +21,14 @@ class StatementFilter(object): self._is_create = False self._begin_depth = 0 + self.consume_ws = False + self.tokens = [] + self.level = 0 + def _change_splitlevel(self, ttype, value): """Get the new split level (increase, decrease or remain equal)""" # PostgreSQL - if ttype == T.Name.Builtin \ - and value.startswith('$') and value.endswith('$'): + if ttype == T.Name.Builtin and value[0] == '$' and value[-1] == '$': # 2nd dbldollar found. $quote$ completed # decrease level @@ -54,6 +56,12 @@ class StatementFilter(object): # returning unified = value.upper() + # three keywords begin with CREATE, but only one of them is DDL + # DDL Create though can contain more words such as "or replace" + if ttype is T.Keyword.DDL and unified.startswith('CREATE'): + self._is_create = True + return 0 + # can have nested declare inside of being... if unified == 'DECLARE' and self._is_create and self._begin_depth == 0: self._in_declare = True @@ -61,14 +69,11 @@ class StatementFilter(object): if unified == 'BEGIN': self._begin_depth += 1 - if self._in_declare or self._is_create: + if self._is_create: # FIXME(andi): This makes no sense. return 1 return 0 - if unified in ('END IF', 'END FOR', 'END WHILE'): - return -1 - # Should this respect a preceeding BEGIN? # In CASE ... WHEN ... END this results in a split level -1. # Would having multiple CASE WHEN END and a Assigment Operator @@ -77,25 +82,19 @@ class StatementFilter(object): self._begin_depth = max(0, self._begin_depth - 1) return -1 - # three keywords begin with CREATE, but only one of them is DDL - # DDL Create though can contain more words such as "or replace" - if ttype is T.Keyword.DDL and unified.startswith('CREATE'): - self._is_create = True - return 0 - - if unified in ('IF', 'FOR', 'WHILE') \ - and self._is_create and self._begin_depth > 0: + if (unified in ('IF', 'FOR', 'WHILE') and + self._is_create and self._begin_depth > 0): return 1 + if unified in ('END IF', 'END FOR', 'END WHILE'): + return -1 + # Default return 0 def process(self, stream): """Process the stream""" - consume_ws = False - splitlevel = 0 - stmt = None - stmt_tokens = [] + EOS_TTYPE = T.Whitespace, T.Comment.Single # Run over all stream tokens for ttype, value in stream: @@ -103,32 +102,22 @@ class StatementFilter(object): # It will count newline token as a non whitespace. In this context # whitespace ignores newlines. # why don't multi line comments also count? - if consume_ws and ttype not in (T.Whitespace, T.Comment.Single): - stmt.tokens = stmt_tokens - yield stmt + if self.consume_ws and ttype not in EOS_TTYPE: + yield sql.Statement(self.tokens) # Reset filter and prepare to process next statement self._reset() - consume_ws = False - splitlevel = 0 - stmt = None - - # Create a new statement if we are not currently in one of them - if stmt is None: - stmt = Statement() - stmt_tokens = [] # Change current split level (increase, decrease or remain equal) - splitlevel += self._change_splitlevel(ttype, value) + self.level += self._change_splitlevel(ttype, value) # Append the token to the current statement - stmt_tokens.append(Token(ttype, value)) + self.tokens.append(sql.Token(ttype, value)) # Check if we get the end of a statement - if splitlevel <= 0 and ttype is T.Punctuation and value == ';': - consume_ws = True + if self.level <= 0 and ttype is T.Punctuation and value == ';': + self.consume_ws = True # Yield pending statement (if any) - if stmt is not None: - stmt.tokens = stmt_tokens - yield stmt + if self.tokens: + yield sql.Statement(self.tokens) -- cgit v1.2.1