2to3: Apply unicode fixer.

The unicode fixer strips the u from u'hi' and converts the unicode type
to str. The first won't work for Python 2 and instead we replace the u
prefix with the sixu function borrowed from the six compatibility
package. That function calls the unicode constructor with the
'unicode_escape' encoder so that the many tests using escaped unicode
characters like u'\u0900' will be handled correctly. That makes the
sixu function a bit different from the asunicode function currently in
numpy.compat and also provides a target that can be converted back to
the u prefix when support for Python 3.2 is dropped. Python 3.3
reintroduced the u prefix for compatibility.

The unicode fixer also replaces 'unicode' with 'str' as 'unicode' is no
longer a builtin in Python 3. For code compatibility, 'unicode' is
defined either as 'str' or 'unicode' in numpy.compat so that checks like

    if isinstance(x, unicode):
        ...

will work properly for all python versions.

Closes #3089.

1 files changed, 9 insertions, 3 deletions

diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py
index ce79edde8..e7ab9bf0f 100644
--- a/numpy/compat/py3k.py
+++ b/numpy/compat/py3k.py
@@ -6,7 +6,7 @@ from __future__ import division, absolute_import, print_function
 
 __all__ = ['bytes', 'asbytes', 'isfileobj', 'getexception', 'strchar',
            'unicode', 'asunicode', 'asbytes_nested', 'asunicode_nested',
-           'asstr', 'open_latin1', 'long', 'basestring']
+           'asstr', 'open_latin1', 'long', 'basestring', 'sixu']
 
 import sys
 
@@ -16,8 +16,8 @@ if sys.version_info[0] >= 3:
     long = int
     integer_types = (int,)
     basestring = str
-    bytes = bytes
     unicode = str
+    bytes = bytes
 
     def asunicode(s):
         if isinstance(s, bytes):
@@ -40,14 +40,17 @@ if sys.version_info[0] >= 3:
     def open_latin1(filename, mode='r'):
         return open(filename, mode=mode, encoding='iso-8859-1')
 
+    def sixu(s):
+        return s
+
     strchar = 'U'
 
 
 else:
     bytes = str
-    unicode = unicode
     long = long
     basestring = basestring
+    unicode = unicode
     integer_types = (int, long)
     asbytes = str
     asstr = str
@@ -65,6 +68,9 @@ else:
     def open_latin1(filename, mode='r'):
         return open(filename, mode=mode)
 
+    def sixu(s):
+        return unicode(s, 'unicode_escape')
+
 
 def getexception():
     return sys.exc_info()[1]