From 264b6d7c46b284492b72ab9e86d9960b3db3bbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 13 Aug 2011 15:48:02 +0200 Subject: Changed ColumnsSelect from yielding tokens to yielding column names (strings) --- sqlparse/filters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index cba7b8f..4d849cb 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -420,7 +420,7 @@ class ColumnsSelect(Filter): elif mode == 1: if value == 'FROM': if oldValue: - yield Name, oldValue + yield oldValue mode = 3 # Columns have been checked @@ -431,7 +431,7 @@ class ColumnsSelect(Filter): elif (token_type == Punctuation and value == ',' and not parenthesis): if oldValue: - yield Name, oldValue + yield oldValue oldValue = "" elif token_type not in Whitespace: @@ -446,7 +446,7 @@ class ColumnsSelect(Filter): elif mode == 2: # We check also for Keywords because a bug in SQLParse if token_type == Name or token_type == Keyword: - yield Name, value + yield value mode = 1 -- cgit v1.2.1 From c88aefb12aa07a2c24ec9cfcbe5b73353b5feed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 18 Aug 2011 00:10:43 +0200 Subject: Added pipeline support maintaining backward compatibility. --- sqlparse/__init__.py | 6 ++++ sqlparse/filters.py | 10 +++++- sqlparse/pipeline.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 sqlparse/pipeline.py (limited to 'sqlparse') diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py index 7698e46..5ccf092 100644 --- a/sqlparse/__init__.py +++ b/sqlparse/__init__.py @@ -53,3 +53,9 @@ def split(sql): stack = engine.FilterStack() stack.split_statements = True return [unicode(stmt) for stmt in stack.run(sql)] + + +from sqlparse.engine.filter import StatementFilter +def split2(stream): + splitter = StatementFilter() + return list(splitter.process(None, stream)) \ No newline at end of file diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 4d849cb..6f9b579 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -463,6 +463,14 @@ class SerializerUnicode(Filter): res += '\n' return res +def Tokens2Unicode(stream): + result = "" + + for _, value in stream: + result += unicode(value) + + return result + class OutputPythonFilter(Filter): @@ -576,4 +584,4 @@ class Limit(Filter): if index and token_type in Keyword and value == 'LIMIT': return stream[4 - index][1] - return -1 + return -1 \ No newline at end of file diff --git a/sqlparse/pipeline.py b/sqlparse/pipeline.py new file mode 100644 index 0000000..167d083 --- /dev/null +++ b/sqlparse/pipeline.py @@ -0,0 +1,90 @@ +# Copyright (C) 2011 Jesus Leganes "piranna", piranna@gmail.com +# +# This module is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +from types import GeneratorType + + +class Pipeline(list): + """Pipeline to process filters sequentially""" + + def __call__(self, stream): + """Run the pipeline + + Return a static (non generator) version of the result + """ + + # Run the stream over all the filters on the pipeline + for filter in self: + # Functions and callable objects (objects with '__call__' method) + if callable(filter): + stream = filter(stream) + + # Normal filters (objects with 'process' method) + else: + stream = filter.process(None, stream) + + # If last filter return a generator, staticalize it inside a list + if isinstance(stream, GeneratorType): + return list(stream) + return stream + + +if __name__ == '__main__': + sql = """-- type: script + -- return: integer + + INCLUDE "Direntry.make.sql"; + + INSERT INTO directories(inode) + VALUES(:inode) + LIMIT 1""" + + sql2 = """SELECT child_entry,asdf AS inode, creation + FROM links + WHERE parent_dir == :parent_dir AND name == :name + LIMIT 1""" + + sql3 = """SELECT + 0 AS st_dev, + 0 AS st_uid, + 0 AS st_gid, + + dir_entries.type AS st_mode, + dir_entries.inode AS st_ino, + COUNT(links.child_entry) AS st_nlink, + + :creation AS st_ctime, + dir_entries.access AS st_atime, + dir_entries.modification AS st_mtime, +-- :creation AS st_ctime, +-- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime, +-- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime, + + COALESCE(files.size,0) AS st_size, -- Python-FUSE + COALESCE(files.size,0) AS size -- PyFilesystem + + FROM dir_entries + LEFT JOIN files + ON dir_entries.inode == files.inode + LEFT JOIN links + ON dir_entries.inode == links.child_entry + + WHERE dir_entries.inode == :inode + + GROUP BY dir_entries.inode + LIMIT 1""" + + from filters import ColumnsSelect + from lexer import tokenize + + def show(args): + for a in args: + print repr(a) + + pipe = Pipeline() + pipe.append(tokenize) + pipe.append(ColumnsSelect()) + + show(pipe(sql3)) \ No newline at end of file -- cgit v1.2.1 From 331a0fe7e84bb7f0e07f21aac1e4df13890b18b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 2 Sep 2011 15:42:09 +0200 Subject: Converted test function of pipeline in standard UnitTest --- sqlparse/pipeline.py | 117 ++++++++++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 52 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/pipeline.py b/sqlparse/pipeline.py index 167d083..11e2e3c 100644 --- a/sqlparse/pipeline.py +++ b/sqlparse/pipeline.py @@ -32,59 +32,72 @@ class Pipeline(list): if __name__ == '__main__': - sql = """-- type: script - -- return: integer - - INCLUDE "Direntry.make.sql"; - - INSERT INTO directories(inode) - VALUES(:inode) - LIMIT 1""" - - sql2 = """SELECT child_entry,asdf AS inode, creation - FROM links - WHERE parent_dir == :parent_dir AND name == :name - LIMIT 1""" - - sql3 = """SELECT - 0 AS st_dev, - 0 AS st_uid, - 0 AS st_gid, - - dir_entries.type AS st_mode, - dir_entries.inode AS st_ino, - COUNT(links.child_entry) AS st_nlink, - - :creation AS st_ctime, - dir_entries.access AS st_atime, - dir_entries.modification AS st_mtime, --- :creation AS st_ctime, --- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime, --- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime, - - COALESCE(files.size,0) AS st_size, -- Python-FUSE - COALESCE(files.size,0) AS size -- PyFilesystem - - FROM dir_entries - LEFT JOIN files - ON dir_entries.inode == files.inode - LEFT JOIN links - ON dir_entries.inode == links.child_entry - - WHERE dir_entries.inode == :inode - - GROUP BY dir_entries.inode - LIMIT 1""" + import unittest from filters import ColumnsSelect from lexer import tokenize - def show(args): - for a in args: - print repr(a) - - pipe = Pipeline() - pipe.append(tokenize) - pipe.append(ColumnsSelect()) - - show(pipe(sql3)) \ No newline at end of file + class Test(unittest.TestCase): + def setUp(self): + self.pipe = Pipeline() + self.pipe.append(tokenize) + self.pipe.append(ColumnsSelect()) + + def test_1(self): + sql = """ + -- type: script + -- return: integer + + INCLUDE "Direntry.make.sql"; + + INSERT INTO directories(inode) + VALUES(:inode) + LIMIT 1""" + self.assertListEqual([], self.pipe(sql)) + + def test_2(self): + sql = """ + SELECT child_entry,asdf AS inode, creation + FROM links + WHERE parent_dir == :parent_dir AND name == :name + LIMIT 1""" + self.assertListEqual([u'child_entry', u'inode', u'creation'], + self.pipe(sql)) + + def test_3(self): + sql = """ + SELECT + 0 AS st_dev, + 0 AS st_uid, + 0 AS st_gid, + + dir_entries.type AS st_mode, + dir_entries.inode AS st_ino, + COUNT(links.child_entry) AS st_nlink, + + :creation AS st_ctime, + dir_entries.access AS st_atime, + dir_entries.modification AS st_mtime, + -- :creation AS st_ctime, + -- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime, + -- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime, + + COALESCE(files.size,0) AS st_size, -- Python-FUSE + COALESCE(files.size,0) AS size -- PyFilesystem + + FROM dir_entries + LEFT JOIN files + ON dir_entries.inode == files.inode + LEFT JOIN links + ON dir_entries.inode == links.child_entry + + WHERE dir_entries.inode == :inode + + GROUP BY dir_entries.inode + LIMIT 1""" + self.assertListEqual([u'st_dev', u'st_uid', u'st_gid', u'st_mode', + u'st_ino', u'st_nlink', u'st_ctime', + u'st_atime', u'st_mtime', u'st_size', u'size'], + self.pipe(sql)) + + unittest.main() \ No newline at end of file -- cgit v1.2.1 From 52817d0626b40614159c1de72c82c60d775a670e Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Sun, 4 Sep 2011 07:45:09 +0200 Subject: Move pipeline test to tests/ directory. --- sqlparse/pipeline.py | 72 ---------------------------------------------------- 1 file changed, 72 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/pipeline.py b/sqlparse/pipeline.py index 11e2e3c..34dad19 100644 --- a/sqlparse/pipeline.py +++ b/sqlparse/pipeline.py @@ -29,75 +29,3 @@ class Pipeline(list): if isinstance(stream, GeneratorType): return list(stream) return stream - - -if __name__ == '__main__': - import unittest - - from filters import ColumnsSelect - from lexer import tokenize - - class Test(unittest.TestCase): - def setUp(self): - self.pipe = Pipeline() - self.pipe.append(tokenize) - self.pipe.append(ColumnsSelect()) - - def test_1(self): - sql = """ - -- type: script - -- return: integer - - INCLUDE "Direntry.make.sql"; - - INSERT INTO directories(inode) - VALUES(:inode) - LIMIT 1""" - self.assertListEqual([], self.pipe(sql)) - - def test_2(self): - sql = """ - SELECT child_entry,asdf AS inode, creation - FROM links - WHERE parent_dir == :parent_dir AND name == :name - LIMIT 1""" - self.assertListEqual([u'child_entry', u'inode', u'creation'], - self.pipe(sql)) - - def test_3(self): - sql = """ - SELECT - 0 AS st_dev, - 0 AS st_uid, - 0 AS st_gid, - - dir_entries.type AS st_mode, - dir_entries.inode AS st_ino, - COUNT(links.child_entry) AS st_nlink, - - :creation AS st_ctime, - dir_entries.access AS st_atime, - dir_entries.modification AS st_mtime, - -- :creation AS st_ctime, - -- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime, - -- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime, - - COALESCE(files.size,0) AS st_size, -- Python-FUSE - COALESCE(files.size,0) AS size -- PyFilesystem - - FROM dir_entries - LEFT JOIN files - ON dir_entries.inode == files.inode - LEFT JOIN links - ON dir_entries.inode == links.child_entry - - WHERE dir_entries.inode == :inode - - GROUP BY dir_entries.inode - LIMIT 1""" - self.assertListEqual([u'st_dev', u'st_uid', u'st_gid', u'st_mode', - u'st_ino', u'st_nlink', u'st_ctime', - u'st_atime', u'st_mtime', u'st_size', u'size'], - self.pipe(sql)) - - unittest.main() \ No newline at end of file -- cgit v1.2.1 From bcaf3269b3a8746a4ee8c848256d5f2b11b25d27 Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Tue, 6 Sep 2011 08:01:12 +0200 Subject: Code cleanup. --- sqlparse/engine/filter.py | 5 +++-- sqlparse/engine/grouping.py | 6 ++++-- sqlparse/sql.py | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/filter.py b/sqlparse/engine/filter.py index 89d9b15..421b3f3 100644 --- a/sqlparse/engine/filter.py +++ b/sqlparse/engine/filter.py @@ -61,14 +61,15 @@ class StatementFilter(TokenFilter): if unified == 'END': # Should this respect a preceeding BEGIN? # In CASE ... WHEN ... END this results in a split level -1. - self._begin_depth = max(0, self._begin_depth-1) + self._begin_depth = max(0, self._begin_depth - 1) return -1 if ttype is T.Keyword.DDL and unified.startswith('CREATE'): self._is_create = True return 0 - if unified in ('IF', 'FOR') and self._is_create and self._begin_depth > 0: + if (unified in ('IF', 'FOR') + and self._is_create and self._begin_depth > 0): return 1 # Default diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 9bc9612..72f919b 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -267,13 +267,15 @@ def group_aliased(tlist): token = tlist.token_next_by_instance(idx, (sql.Identifier, sql.Function)) while token: next_ = tlist.token_next(tlist.token_index(token)) - if next_ is not None and isinstance(next_, (sql.Identifier, sql.Function)): + if next_ is not None and isinstance(next_, + (sql.Identifier, sql.Function)): grp = tlist.tokens_between(token, next_)[1:] token.tokens.extend(grp) for t in grp: tlist.tokens.remove(t) idx = tlist.token_index(token) + 1 - token = tlist.token_next_by_instance(idx, (sql.Identifier, sql.Function)) + token = tlist.token_next_by_instance(idx, + (sql.Identifier, sql.Function)) def group_typecasts(tlist): diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 5b8f067..4d56bf3 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -146,7 +146,7 @@ class TokenList(Token): def _pprint_tree(self, max_depth=None, depth=0): """Pretty-print the object tree.""" - indent = ' '*(depth*2) + indent = ' ' * (depth * 2) for idx, token in enumerate(self.tokens): if token.is_group(): pre = ' +-' @@ -156,7 +156,7 @@ class TokenList(Token): token._get_repr_name(), token._get_repr_value()) if (token.is_group() and (max_depth is None or depth < max_depth)): - token._pprint_tree(max_depth, depth+1) + token._pprint_tree(max_depth, depth + 1) def flatten(self): """Generator yielding ungrouped tokens. -- cgit v1.2.1