- unicode literals need to just be handled differently if they have utf-8

encoded in them vs. unicode escaping.  not worth figuring out how to combine
these right now

6 files changed, 59 insertions, 47 deletions

diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py
index d742a71b5..1ca8f4e64 100644
--- a/lib/sqlalchemy/dialects/sqlite/base.py
+++ b/lib/sqlalchemy/dialects/sqlite/base.py
@@ -812,7 +812,7 @@ class SQLiteDialect(default.DefaultDialect):
             coltype = sqltypes.NullType()
 
         if default is not None:
-            default = str(default)
+            default = util.text_type(default)
 
         return {
             'name': name,
diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index d51dd625a..c3aea159a 100644
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -1868,11 +1868,11 @@ class DDLCompiler(engine.Compiled):
                     first_pk = True
             except exc.CompileError as ce:
                 util.raise_from_cause(
-                    exc.CompileError(util.u("(in table '%s', column '%s'): %s" % (
+                    exc.CompileError(util.u("(in table '%s', column '%s'): %s") % (
                                                 table.description,
                                                 column.name,
                                                 ce.args[0]
-                                            ))))
+                                            )))
 
         const = self.create_table_constraints(table)
         if const:
@@ -2344,7 +2344,7 @@ class IdentifierPreparer(object):
         lc_value = value.lower()
         return (lc_value in self.reserved_words
                 or value[0] in self.illegal_initial_characters
-                or not self.legal_characters.match(str(value))
+                or not self.legal_characters.match(util.text_type(value))
                 or (lc_value != value))
 
     def quote_schema(self, schema, force):
diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py
index 72baa9f36..b1f1a2db6 100644
--- a/lib/sqlalchemy/util/__init__.py
+++ b/lib/sqlalchemy/util/__init__.py
@@ -8,7 +8,7 @@ from .compat import callable, cmp, reduce,  \
     threading, py3k, py2k, jython, pypy, cpython, win32, \
     pickle, dottedgetter, parse_qsl, namedtuple, next, WeakSet, reraise, \
     raise_from_cause, text_type, string_types, int_types, binary_type, \
-    quote_plus, with_metaclass, print_, itertools_filterfalse, u, b,\
+    quote_plus, with_metaclass, print_, itertools_filterfalse, u, ue, b,\
     unquote_plus
 
 from ._collections import KeyedTuple, ImmutableContainer, immutabledict, \
diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py
index bc7a0fe21..94b35f019 100644
--- a/lib/sqlalchemy/util/compat.py
+++ b/lib/sqlalchemy/util/compat.py
@@ -46,6 +46,9 @@ if py3k:
     def u(s):
         return s
 
+    def ue(s):
+        return s
+
     def b(s):
         return s.encode("latin-1")
 
@@ -79,6 +82,13 @@ else:
         return (ord(byte) for byte in buf)
 
     def u(s):
+        # this differs from what six does, which doesn't support non-ASCII
+        # strings - we only use u() with
+        # literal source strings, and all our source files with non-ascii
+        # in them (all are tests) are utf-8 encoded.
+        return unicode(s, "utf-8")
+
+    def ue(s):
         return unicode(s, "unicode_escape")
 
     def b(s):
diff --git a/test/sql/test_compiler.py b/test/sql/test_compiler.py
index e3f7df702..9cb0e6bbe 100644
--- a/test/sql/test_compiler.py
+++ b/test/sql/test_compiler.py
@@ -20,6 +20,7 @@ from sqlalchemy import Integer, String, MetaData, Table, Column, select, \
     intersect, union_all, Boolean, distinct, join, outerjoin, asc, desc,\
     over, subquery, case
 import decimal
+from sqlalchemy.util import u
 from sqlalchemy import exc, sql, util, types, schema
 from sqlalchemy.sql import table, column, label
 from sqlalchemy.sql.expression import ClauseList, _literal_as_text, HasPrefixes
@@ -2680,11 +2681,11 @@ class DDLTest(fixtures.TestBase, AssertsCompiledSQL):
     def test_reraise_of_column_spec_issue_unicode(self):
         MyType = self._illegal_type_fixture()
         t1 = Table('t', MetaData(),
-            Column('méil', MyType())
+            Column(u('méil'), MyType())
         )
         assert_raises_message(
             exc.CompileError,
-            r"\(in table 't', column 'méil'\): Couldn't compile type",
+            u(r"\(in table 't', column 'méil'\): Couldn't compile type"),
             schema.CreateTable(t1).compile
         )
 
diff --git a/test/sql/test_unicode.py b/test/sql/test_unicode.py
index 2ab081dad..ffcef903f 100644
--- a/test/sql/test_unicode.py
+++ b/test/sql/test_unicode.py
@@ -7,6 +7,7 @@ from sqlalchemy import testing
 from sqlalchemy.testing.engines import utf8_engine
 from sqlalchemy.sql import column
 from sqlalchemy.testing.schema import Table, Column
+from sqlalchemy.util import u, ue
 
 class UnicodeSchemaTest(fixtures.TestBase):
     __requires__ = ('unicode_ddl',)
@@ -18,14 +19,14 @@ class UnicodeSchemaTest(fixtures.TestBase):
         unicode_bind = utf8_engine()
 
         metadata = MetaData(unicode_bind)
-        t1 = Table('unitable1', metadata,
-            Column('méil', Integer, primary_key=True),
-            Column('\u6e2c\u8a66', Integer),
+        t1 = Table(u('unitable1'), metadata,
+            Column(u('méil'), Integer, primary_key=True),
+            Column(ue('\u6e2c\u8a66'), Integer),
             test_needs_fk=True,
             )
-        t2 = Table('Unitéble2', metadata,
-            Column('méil', Integer, primary_key=True, key="a"),
-            Column('\u6e2c\u8a66', Integer, ForeignKey('unitable1.méil'),
+        t2 = Table(u('Unitéble2'), metadata,
+            Column(u('méil'), Integer, primary_key=True, key="a"),
+            Column(ue('\u6e2c\u8a66'), Integer, ForeignKey(u('unitable1.méil')),
                    key="b"
                    ),
                    test_needs_fk=True,
@@ -33,27 +34,27 @@ class UnicodeSchemaTest(fixtures.TestBase):
 
         # Few DBs support Unicode foreign keys
         if testing.against('sqlite'):
-            t3 = Table('\u6e2c\u8a66', metadata,
-                       Column('\u6e2c\u8a66_id', Integer, primary_key=True,
+            t3 = Table(ue('\u6e2c\u8a66'), metadata,
+                       Column(ue('\u6e2c\u8a66_id'), Integer, primary_key=True,
                               autoincrement=False),
-                       Column('unitable1_\u6e2c\u8a66', Integer,
-                              ForeignKey('unitable1.\u6e2c\u8a66')
+                       Column(ue('unitable1_\u6e2c\u8a66'), Integer,
+                              ForeignKey(ue('unitable1.\u6e2c\u8a66'))
                               ),
-                       Column('Unitéble2_b', Integer,
-                              ForeignKey('Unitéble2.b')
+                       Column(u('Unitéble2_b'), Integer,
+                              ForeignKey(u('Unitéble2.b'))
                               ),
-                       Column('\u6e2c\u8a66_self', Integer,
-                              ForeignKey('\u6e2c\u8a66.\u6e2c\u8a66_id')
+                       Column(ue('\u6e2c\u8a66_self'), Integer,
+                              ForeignKey(ue('\u6e2c\u8a66.\u6e2c\u8a66_id'))
                               ),
                        test_needs_fk=True,
                        )
         else:
-            t3 = Table('\u6e2c\u8a66', metadata,
-                       Column('\u6e2c\u8a66_id', Integer, primary_key=True,
+            t3 = Table(ue('\u6e2c\u8a66'), metadata,
+                       Column(ue('\u6e2c\u8a66_id'), Integer, primary_key=True,
                               autoincrement=False),
-                       Column('unitable1_\u6e2c\u8a66', Integer),
-                       Column('Unitéble2_b', Integer),
-                       Column('\u6e2c\u8a66_self', Integer),
+                       Column(ue('unitable1_\u6e2c\u8a66'), Integer),
+                       Column(u('Unitéble2_b'), Integer),
+                       Column(ue('\u6e2c\u8a66_self'), Integer),
                        test_needs_fk=True,
                        )
         metadata.create_all()
@@ -72,42 +73,42 @@ class UnicodeSchemaTest(fixtures.TestBase):
         del unicode_bind
 
     def test_insert(self):
-        t1.insert().execute({'méil':1, '\u6e2c\u8a66':5})
-        t2.insert().execute({'a':1, 'b':1})
-        t3.insert().execute({'\u6e2c\u8a66_id': 1,
-                             'unitable1_\u6e2c\u8a66': 5,
-                             'Unitéble2_b': 1,
-                             '\u6e2c\u8a66_self': 1})
+        t1.insert().execute({u('méil'):1, ue('\u6e2c\u8a66'):5})
+        t2.insert().execute({u('a'):1, u('b'):1})
+        t3.insert().execute({ue('\u6e2c\u8a66_id'): 1,
+                             ue('unitable1_\u6e2c\u8a66'): 5,
+                             u('Unitéble2_b'): 1,
+                             ue('\u6e2c\u8a66_self'): 1})
 
         assert t1.select().execute().fetchall() == [(1, 5)]
         assert t2.select().execute().fetchall() == [(1, 1)]
         assert t3.select().execute().fetchall() == [(1, 5, 1, 1)]
 
     def test_reflect(self):
-        t1.insert().execute({'méil':2, '\u6e2c\u8a66':7})
-        t2.insert().execute({'a':2, 'b':2})
-        t3.insert().execute({'\u6e2c\u8a66_id': 2,
-                             'unitable1_\u6e2c\u8a66': 7,
-                             'Unitéble2_b': 2,
-                             '\u6e2c\u8a66_self': 2})
+        t1.insert().execute({u('méil'):2, ue('\u6e2c\u8a66'):7})
+        t2.insert().execute({u('a'):2, u('b'):2})
+        t3.insert().execute({ue('\u6e2c\u8a66_id'): 2,
+                             ue('unitable1_\u6e2c\u8a66'): 7,
+                             u('Unitéble2_b'): 2,
+                             ue('\u6e2c\u8a66_self'): 2})
 
         meta = MetaData(unicode_bind)
         tt1 = Table(t1.name, meta, autoload=True)
         tt2 = Table(t2.name, meta, autoload=True)
         tt3 = Table(t3.name, meta, autoload=True)
 
-        tt1.insert().execute({'méil':1, '\u6e2c\u8a66':5})
-        tt2.insert().execute({'méil':1, '\u6e2c\u8a66':1})
-        tt3.insert().execute({'\u6e2c\u8a66_id': 1,
-                              'unitable1_\u6e2c\u8a66': 5,
-                              'Unitéble2_b': 1,
-                              '\u6e2c\u8a66_self': 1})
+        tt1.insert().execute({u('méil'):1, ue('\u6e2c\u8a66'):5})
+        tt2.insert().execute({u('méil'):1, ue('\u6e2c\u8a66'):1})
+        tt3.insert().execute({ue('\u6e2c\u8a66_id'): 1,
+                              ue('unitable1_\u6e2c\u8a66'): 5,
+                              u('Unitéble2_b'): 1,
+                              ue('\u6e2c\u8a66_self'): 1})
 
-        self.assert_(tt1.select(order_by=desc('méil')).execute().fetchall() ==
+        self.assert_(tt1.select(order_by=desc(u('méil'))).execute().fetchall() ==
                      [(2, 7), (1, 5)])
-        self.assert_(tt2.select(order_by=desc('méil')).execute().fetchall() ==
+        self.assert_(tt2.select(order_by=desc(u('méil'))).execute().fetchall() ==
                      [(2, 2), (1, 1)])
-        self.assert_(tt3.select(order_by=desc('\u6e2c\u8a66_id')).
+        self.assert_(tt3.select(order_by=desc(ue('\u6e2c\u8a66_id'))).
                      execute().fetchall() ==
                      [(2, 7, 2, 2), (1, 5, 1, 1)])
         meta.drop_all()