Fix SerializerUnicode to split unquoted newlines

This provides a fix to issue #131. The `split_unquoted_newlines()`
function added to the utils module handles the splitting of the string
by performing a simple iteration of the string passed in and splitting
on unquoted CR, LF, or CR+LFs as they are found.

2 files changed, 25 insertions, 1 deletions

diff --git a/tests/test_format.py b/tests/test_format.py
index 701540b..b77b7a1 100644
--- a/tests/test_format.py
+++ b/tests/test_format.py
@@ -77,6 +77,23 @@ class TestFormat(TestCaseBase):
         s = 'select\n* /* foo */  from bar '
         self.ndiffAssertEqual(f(s), 'select * /* foo */ from bar')
 
+    def test_notransform_of_quoted_crlf(self):
+        # Make sure that CR/CR+LF characters inside string literals don't get
+        # affected by the formatter.
+
+        s1 = "SELECT some_column LIKE 'value\r'"
+        s2 = "SELECT some_column LIKE 'value\r'\r\nWHERE id = 1\n"
+        s3 = "SELECT some_column LIKE 'value\\'\r' WHERE id = 1\r"
+        s4 = "SELECT some_column LIKE 'value\\\\\\'\r' WHERE id = 1\r\n"
+
+        f = lambda x: sqlparse.format(x)
+
+        # Because of the use of
+        self.ndiffAssertEqual(f(s1), "SELECT some_column LIKE 'value\r'")
+        self.ndiffAssertEqual(f(s2), "SELECT some_column LIKE 'value\r'\nWHERE id = 1\n")
+        self.ndiffAssertEqual(f(s3), "SELECT some_column LIKE 'value\\'\r' WHERE id = 1\n")
+        self.ndiffAssertEqual(f(s4), "SELECT some_column LIKE 'value\\\\\\'\r' WHERE id = 1\n")
+
     def test_outputformat(self):
         sql = 'select * from foo;'
         self.assertRaises(SQLParseError, sqlparse.format, sql,
diff --git a/tests/utils.py b/tests/utils.py
index e2c01a3..9eb46bf 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -8,6 +8,8 @@ import os
 import unittest
 from StringIO import StringIO
 
+import sqlparse.utils
+
 NL = '\n'
 DIR_PATH = os.path.abspath(os.path.dirname(__file__))
 PARENT_DIR = os.path.dirname(DIR_PATH)
@@ -31,7 +33,12 @@ class TestCaseBase(unittest.TestCase):
         if first != second:
             sfirst = unicode(first)
             ssecond = unicode(second)
-            diff = difflib.ndiff(sfirst.splitlines(), ssecond.splitlines())
+            # Using the built-in .splitlines() method here will cause incorrect
+            # results when splitting statements that have quoted CR/CR+LF
+            # characters.
+            sfirst = sqlparse.utils.split_unquoted_newlines(sfirst)
+            ssecond = sqlparse.utils.split_unquoted_newlines(ssecond)
+            diff = difflib.ndiff(sfirst, ssecond)
             fp = StringIO()
             fp.write(NL)
             fp.write(NL.join(diff))