Handle whitepaces between operators correctly, improve handling of concatenated strings (issue53).

3 files changed, 36 insertions, 2 deletions

diff --git a/CHANGES b/CHANGES
index 3e6e071..25eecb3 100644
--- a/CHANGES
+++ b/CHANGES
@@ -3,6 +3,8 @@ Development Version
 
 Bug Fixes
  * Improve handling of quoted identifiers (issue78).
+ * Improve grouping and formatting of identifiers with operators (issue53).
+ * Improve grouping and formatting of concatenated strings (issue53).
 
 Other
  * Deprecated sqlparse.SQLParseError. Please use
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index f30d9e1..a98e787 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -138,10 +138,15 @@ def group_identifier(tlist):
                        or y.ttype is T.Operator
                        or y.ttype is T.Wildcard),
             lambda y: (y.ttype in (T.String.Symbol,
+                                   T.String.Single,
                                    T.Name,
                                    T.Wildcard,
                                    T.Literal.Number.Integer))))
         for t in tl.tokens[i:]:
+            # Don't take whitespaces into account.
+            if t.ttype is T.Whitespace:
+                yield t
+                continue
             if next(x)(t):
                 yield t
             else:
@@ -150,7 +155,7 @@ def group_identifier(tlist):
     def _next_token(tl, i):
         # chooses the next token. if two tokens are found then the
         # first is returned.
-        t1 = tl.token_next_by_type(i, (T.String.Symbol, T.Name))
+        t1 = tl.token_next_by_type(i, (T.String.Symbol, T.String.Single, T.Name))
         t2 = tl.token_next_by_instance(i, sql.Function)
         if t1 and t2:
             i1 = tl.token_index(t1)
@@ -175,6 +180,9 @@ def group_identifier(tlist):
         identifier_tokens = [token] + list(
             _consume_cycle(tlist,
                            tlist.token_index(token) + 1))
+        # remove trailing whitespace
+        if identifier_tokens and identifier_tokens[-1].ttype is T.Whitespace:
+            identifier_tokens = identifier_tokens[:-1]
         if not (len(identifier_tokens) == 1
                 and isinstance(identifier_tokens[0], sql.Function)):
             group = tlist.group_tokens(sql.Identifier, identifier_tokens)
diff --git a/tests/test_grouping.py b/tests/test_grouping.py
index c63d8e5..345e62f 100644
--- a/tests/test_grouping.py
+++ b/tests/test_grouping.py
@@ -192,7 +192,7 @@ class TestGrouping(TestCaseBase):
     def test_varchar(self):
         p = sqlparse.parse('"text" Varchar(50) NOT NULL')[0]
         self.assert_(isinstance(p.tokens[2], sql.Function))
-        
+
 
 class TestStatement(TestCaseBase):
 
@@ -207,3 +207,27 @@ class TestStatement(TestCaseBase):
         # are parsed as two statements where later only consists of the
         # trailing whitespace.
         self.assertEqual(f('\n').get_type(), 'UNKNOWN')
+
+
+def test_identifier_with_operators():  # issue 53
+    p = sqlparse.parse('foo||bar')[0]
+    assert len(p.tokens) == 1
+    assert isinstance(p.tokens[0], sql.Identifier)
+    # again with whitespaces
+    p = sqlparse.parse('foo || bar')[0]
+    assert len(p.tokens) == 1
+    assert isinstance(p.tokens[0], sql.Identifier)
+
+
+def test_identifier_with_op_trailing_ws():
+    # make sure trailing whitespace isn't grouped with identifier
+    p = sqlparse.parse('foo || bar ')[0]
+    assert len(p.tokens) == 2
+    assert isinstance(p.tokens[0], sql.Identifier)
+    assert p.tokens[1].ttype is T.Whitespace
+
+
+def test_identifier_string_concat():
+    p = sqlparse.parse('\'foo\' || bar')[0]
+    assert len(p.tokens) == 1
+    assert isinstance(p.tokens[0], sql.Identifier)