Minor clarifications and doc updates

2 files changed, 12 insertions, 10 deletions

diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py
index bacaa78..e622292 100644
--- a/sqlparse/__init__.py
+++ b/sqlparse/__init__.py
@@ -54,14 +54,14 @@ def split(sql):
     stack.split_statements = True
     return [unicode(stmt) for stmt in stack.run(sql)]
 
-def splitstream(sql):
-    """Split *sql* into single statements.
+def splitstream(stream):
+    """Split sql statements from file-like object .
 
-    Returns a list of strings.
+    Returns a list of Statement objects.
     """
     stack = engine.FilterStack()
     stack.split_statements = True
-    return stack.run(sql)
+    return stack.run(stream)
 
 from sqlparse.engine.filter import StatementFilter
 def split2(stream):
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 82a6169..a0ce5d2 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -279,6 +279,7 @@ class Lexer(object):
                 m = rexmatch(text, pos)
                 if m:
                     if hasmore and m.end() == len(text):
+                        # Since this is end, token may be truncated
                         continue
 
                     # print rex.pattern
@@ -315,13 +316,14 @@ class Lexer(object):
                         statetokens = tokendefs[statestack[-1]]
                     break
             else:
+                if hasmore:
+                    buf = self._decode(stream.read(self.bufsize))
+                    hasmore = len(buf) == self.bufsize
+                    text = text[pos:] + buf
+                    pos = 0
+                    continue
+
                 try:
-                    if hasmore:
-                        buf = self._decode(stream.read(self.bufsize))
-                        hasmore = len(buf) == self.bufsize
-                        text = text[pos:] + buf
-                        pos = 0
-                        continue
                     if text[pos] == '\n':
                         # at EOL, reset state to "root"
                         pos += 1