Merged.

11 files changed, 132 insertions, 19 deletions

diff --git a/TODO b/TODO
index c222c4f..166df20 100644
--- a/TODO
+++ b/TODO
@@ -4,5 +4,4 @@
 * Provide a function to replace tokens. See this thread: https://groups.google.com/d/msg/sqlparse/5xmBL2UKqX4/ZX9z_peve-AJ
 * Fix bugs on issue tracker.
 * Document filter stack and processing phases.
-* Rewrite App Engine application using webapp.
 * See KnownIssues http://code.google.com/p/python-sqlparse/wiki/KnownIssues
diff --git a/extras/appengine/app.yaml b/extras/appengine/app.yaml
index 3afa891..6c1de16 100644
--- a/extras/appengine/app.yaml
+++ b/extras/appengine/app.yaml
@@ -15,10 +15,9 @@ handlers:
   static_files: static/robots.txt
   upload: static/robots.txt
 
-- url: /static/(script.js|styles.css|upload.py)
+- url: /static/(script.js|styles.css)
   static_files: static/\1
-  upload: static/(script.js|styles.css|upload.py)
-  expiration: 1h  # Shorter expiration, these change often
+  upload: static/(script.js|styles.css)
 
 - url: /static
   static_dir: static
diff --git a/extras/appengine/sqlformat/__init__.py b/extras/appengine/sqlformat/__init__.py
index db9c132..11f4d9d 100644
--- a/extras/appengine/sqlformat/__init__.py
+++ b/extras/appengine/sqlformat/__init__.py
@@ -14,11 +14,6 @@ def ping():
 def warmup():
     return make_response('polishing chrome')
 
-@app.route('/fail')
-def fail():
-    # test URL for failure handling
-    raise AssertionError('You shouldn\'t be here!')
-
 
 # Register legacy URLs last so that newer URLs replace them.
 app.register_blueprint(legacy)
diff --git a/extras/appengine/sqlformat/legacy.py b/extras/appengine/sqlformat/legacy.py
index 6d3aaf8..b563a38 100644
--- a/extras/appengine/sqlformat/legacy.py
+++ b/extras/appengine/sqlformat/legacy.py
@@ -1,5 +1,8 @@
 """Legacy URLs."""
 
+# This module reflects the URLs and behavior of the former Django
+# application.
+
 import logging
 import os
 import time
diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py
index 7698e46..5ccf092 100644
--- a/sqlparse/__init__.py
+++ b/sqlparse/__init__.py
@@ -53,3 +53,9 @@ def split(sql):
     stack = engine.FilterStack()
     stack.split_statements = True
     return [unicode(stmt) for stmt in stack.run(sql)]
+
+
+from sqlparse.engine.filter import StatementFilter
+def split2(stream):
+    splitter = StatementFilter()
+    return list(splitter.process(None, stream))
\ No newline at end of file
diff --git a/sqlparse/engine/filter.py b/sqlparse/engine/filter.py
index 89d9b15..421b3f3 100644
--- a/sqlparse/engine/filter.py
+++ b/sqlparse/engine/filter.py
@@ -61,14 +61,15 @@ class StatementFilter(TokenFilter):
         if unified == 'END':
             # Should this respect a preceeding BEGIN?
             # In CASE ... WHEN ... END this results in a split level -1.
-            self._begin_depth = max(0, self._begin_depth-1)
+            self._begin_depth = max(0, self._begin_depth - 1)
             return -1
 
         if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
             self._is_create = True
             return 0
 
-        if unified in ('IF', 'FOR') and self._is_create and self._begin_depth > 0:
+        if (unified in ('IF', 'FOR')
+            and self._is_create and self._begin_depth > 0):
             return 1
 
         # Default
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index 6e99782..cc75de4 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -280,13 +280,15 @@ def group_aliased(tlist):
     token = tlist.token_next_by_instance(idx, (sql.Identifier, sql.Function))
     while token:
         next_ = tlist.token_next(tlist.token_index(token))
-        if next_ is not None and isinstance(next_, (sql.Identifier, sql.Function)):
+        if next_ is not None and isinstance(next_,
+                                            (sql.Identifier, sql.Function)):
             grp = tlist.tokens_between(token, next_)[1:]
             token.tokens.extend(grp)
             for t in grp:
                 tlist.tokens.remove(t)
         idx = tlist.token_index(token) + 1
-        token = tlist.token_next_by_instance(idx, (sql.Identifier, sql.Function))
+        token = tlist.token_next_by_instance(idx,
+                                             (sql.Identifier, sql.Function))
 
 
 def group_typecasts(tlist):
diff --git a/sqlparse/filters.py b/sqlparse/filters.py
index cba7b8f..6f9b579 100644
--- a/sqlparse/filters.py
+++ b/sqlparse/filters.py
@@ -420,7 +420,7 @@ class ColumnsSelect(Filter):
             elif mode == 1:
                 if value == 'FROM':
                     if oldValue:
-                        yield Name, oldValue
+                        yield oldValue
 
                     mode = 3    # Columns have been checked
 
@@ -431,7 +431,7 @@ class ColumnsSelect(Filter):
                 elif (token_type == Punctuation
                       and value == ',' and not parenthesis):
                     if oldValue:
-                        yield Name, oldValue
+                        yield oldValue
                     oldValue = ""
 
                 elif token_type not in Whitespace:
@@ -446,7 +446,7 @@ class ColumnsSelect(Filter):
             elif mode == 2:
                 # We check also for Keywords because a bug in SQLParse
                 if token_type == Name or token_type == Keyword:
-                    yield Name, value
+                    yield value
                     mode = 1
 
 
@@ -463,6 +463,14 @@ class SerializerUnicode(Filter):
             res += '\n'
         return res
 
+def Tokens2Unicode(stream):
+    result = ""
+
+    for _, value in stream:
+        result += unicode(value)
+
+    return result
+
 
 class OutputPythonFilter(Filter):
 
@@ -576,4 +584,4 @@ class Limit(Filter):
             if index and token_type in Keyword and value == 'LIMIT':
                 return stream[4 - index][1]
 
-        return -1
+        return -1
\ No newline at end of file
diff --git a/sqlparse/pipeline.py b/sqlparse/pipeline.py
new file mode 100644
index 0000000..34dad19
--- /dev/null
+++ b/sqlparse/pipeline.py
@@ -0,0 +1,31 @@
+# Copyright (C) 2011 Jesus Leganes "piranna", piranna@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+from types import GeneratorType
+
+
+class Pipeline(list):
+    """Pipeline to process filters sequentially"""
+
+    def __call__(self, stream):
+        """Run the pipeline
+
+        Return a static (non generator) version of the result
+        """
+
+        # Run the stream over all the filters on the pipeline
+        for filter in self:
+            # Functions and callable objects (objects with '__call__' method)
+            if callable(filter):
+                stream = filter(stream)
+
+            # Normal filters (objects with 'process' method)
+            else:
+                stream = filter.process(None, stream)
+
+        # If last filter return a generator, staticalize it inside a list
+        if isinstance(stream, GeneratorType):
+            return list(stream)
+        return stream
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index 5b8f067..4d56bf3 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -146,7 +146,7 @@ class TokenList(Token):
 
     def _pprint_tree(self, max_depth=None, depth=0):
         """Pretty-print the object tree."""
-        indent = ' '*(depth*2)
+        indent = ' ' * (depth * 2)
         for idx, token in enumerate(self.tokens):
             if token.is_group():
                 pre = ' +-'
@@ -156,7 +156,7 @@ class TokenList(Token):
                                         token._get_repr_name(),
                                         token._get_repr_value())
             if (token.is_group() and (max_depth is None or depth < max_depth)):
-                token._pprint_tree(max_depth, depth+1)
+                token._pprint_tree(max_depth, depth + 1)
 
     def flatten(self):
         """Generator yielding ungrouped tokens.
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
new file mode 100644
index 0000000..56eed4a
--- /dev/null
+++ b/tests/test_pipeline.py
@@ -0,0 +1,69 @@
+import unittest
+
+from sqlparse.filters import ColumnsSelect
+from sqlparse.lexer import tokenize
+from sqlparse.pipeline import Pipeline
+
+class Test(unittest.TestCase):
+
+    def setUp(self):
+        self.pipe = Pipeline()
+        self.pipe.append(tokenize)
+        self.pipe.append(ColumnsSelect())
+
+    def test_1(self):
+        sql = """
+        -- type: script
+        -- return: integer
+
+        INCLUDE "Direntry.make.sql";
+
+        INSERT INTO directories(inode)
+        VALUES(:inode)
+        LIMIT 1"""
+        self.assertEqual([], self.pipe(sql))
+
+    def test_2(self):
+        sql = """
+        SELECT child_entry,asdf AS inode, creation
+        FROM links
+        WHERE parent_dir == :parent_dir AND name == :name
+        LIMIT 1"""
+        self.assertEqual([u'child_entry', u'inode', u'creation'],
+                         self.pipe(sql))
+
+    def test_3(self):
+        sql = """
+        SELECT
+        0 AS st_dev,
+        0 AS st_uid,
+        0 AS st_gid,
+
+        dir_entries.type         AS st_mode,
+        dir_entries.inode        AS st_ino,
+        COUNT(links.child_entry) AS st_nlink,
+
+        :creation                AS st_ctime,
+        dir_entries.access       AS st_atime,
+        dir_entries.modification AS st_mtime,
+        --    :creation                                                AS st_ctime,
+        --    CAST(STRFTIME('%s',dir_entries.access)       AS INTEGER) AS st_atime,
+        --    CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime,
+
+        COALESCE(files.size,0) AS st_size, -- Python-FUSE
+        COALESCE(files.size,0) AS size     -- PyFilesystem
+
+        FROM dir_entries
+        LEFT JOIN files
+        ON dir_entries.inode == files.inode
+        LEFT JOIN links
+        ON dir_entries.inode == links.child_entry
+
+        WHERE dir_entries.inode == :inode
+
+        GROUP BY dir_entries.inode
+        LIMIT 1"""
+        self.assertEqual([u'st_dev', u'st_uid', u'st_gid', u'st_mode',
+                          u'st_ino', u'st_nlink', u'st_ctime',
+                          u'st_atime', u'st_mtime', u'st_size', u'size'],
+                         self.pipe(sql))