import cleanups

* moved bytes compat functions from utils to utils.compat
  (bord, bjoin, bjoin_ints, bjoin_elems, ujoin)
* renamed bord -> belem_ord for clarify
* a bunch of to_native_str() always use ascii, and
  have fixed input types (always bytes or always unicode).
  these don't need overhead of to_native_str(), so replaced
  those calls with two new funcs: compat.bascii_to_str() /
  compat.uascii_to_str()
* cleaned up a lot of imports from utils/utils.compat to
  pull from correct module
* simplified the to_string() logic of a bunch of handlers
  to reduce unicode<->byte transitions

4 files changed, 51 insertions, 45 deletions

diff --git a/passlib/tests/test_context.py b/passlib/tests/test_context.py
index bb2f78c..0e5d637 100644
--- a/passlib/tests/test_context.py
+++ b/passlib/tests/test_context.py
@@ -18,7 +18,7 @@ except ImportError:
 #pkg
 from passlib import hash
 from passlib.context import CryptContext, CryptPolicy, LazyCryptContext
-from passlib.utils import to_bytes, to_unicode, PasslibPolicyWarning, tick
+from passlib.utils import tick, to_bytes, to_unicode, PasslibPolicyWarning
 from passlib.utils.compat import irange, u
 import passlib.utils.handlers as uh
 from passlib.tests.utils import TestCase, mktemp, catch_warnings, \
diff --git a/passlib/tests/test_utils.py b/passlib/tests/test_utils.py
index e04d65f..a4e021d 100644
--- a/passlib/tests/test_utils.py
+++ b/passlib/tests/test_utils.py
@@ -11,8 +11,8 @@ import warnings
 #site
 #pkg
 #module
-from passlib.utils import bytes, b, to_native_str
-from passlib.utils.compat import unicode, PY3, u
+from passlib.utils.compat import b, bytes, bascii_to_str, irange, PY2, PY3, u, \
+                                 unicode
 from passlib.tests.utils import TestCase, Params as ak, enable_option, catch_warnings
 
 def hb(source):
@@ -387,30 +387,32 @@ class CodecTest(TestCase):
         "test to_native_str()"
         from passlib.utils import to_native_str
 
-        # ascii goes to native string w/o problems
-        self.assertEqual(to_native_str(u('abc')), 'abc')
-        self.assertEqual(to_native_str(b('abc')), 'abc')
+        # test plain ascii
+        self.assertEqual(to_native_str(u('abc', 'ascii')), 'abc')
+        self.assertEqual(to_native_str(b('abc', 'ascii')), 'abc')
 
-        # other types rejected
-        self.assertRaises(TypeError, to_native_str, None)
-
-        # non-ascii unicode should throw error under py2, unless codec specified
+        # test invalid ascii
         if PY3:
-            self.assertEqual(to_native_str(u('\x00\xff')), '\x00\xff')
+            self.assertEqual(to_native_str(u('\xE0'), 'ascii'), '\xE0')
+            self.assertRaises(UnicodeDecodeError, to_native_str, b('\xC3\xA0'),
+                              'ascii')
         else:
-            self.assertEqual(to_native_str(u('\x00\xff')), '\x00\xc3\xbf')
-            self.assertRaises(UnicodeEncodeError, to_native_str, u('\x00\xff'),
+            self.assertRaises(UnicodeEncodeError, to_native_str, u('\xE0'),
                               'ascii')
+            self.assertEqual(to_native_str(b('\xC3\xA0'), 'ascii'), '\xC3\xA0')
 
-        # non-ascii bytes should throw error under py3, unless codec specified
-        if PY3:
-            self.assertRaises(UnicodeDecodeError, to_native_str, b('\x00\xff'))
-        else:
-            self.assertEqual(to_native_str(b('\x00\xff')), '\x00\xff')
+        # test latin-1
+        self.assertEqual(to_native_str(u('\xE0'), 'latin-1'), '\xE0')
+        self.assertEqual(to_native_str(b('\xE0'), 'latin-1'), '\xE0')
+
+        # test utf-8
+        self.assertEqual(to_native_str(u('\xE0'), 'utf-8'),
+                         '\xE0' if PY3 else '\xC3\xA0')
+        self.assertEqual(to_native_str(b('\xC3\xA0'), 'utf-8'),
+                         '\xE0' if PY3 else '\xC3\xA0')
 
-        # latin-1 should work if explicitly chosen
-        self.assertEqual(to_native_str(u('\x00\xff'), 'latin-1'), '\x00\xff')
-        self.assertEqual(to_native_str(b('\x00\xff'), 'latin-1'), '\x00\xff')
+        # other types rejected
+        self.assertRaises(TypeError, to_native_str, None, 'ascii')
 
     def test_is_ascii_safe(self):
         "test is_ascii_safe()"
@@ -894,7 +896,7 @@ class _MD4_Test(TestCase):
         "test md4 digest()"
         md4 = self.hash
         for input, hex in self.vectors:
-            out = to_native_str(hexlify(md4(input).digest()))
+            out = bascii_to_str(hexlify(md4(input).digest()))
             self.assertEqual(out, hex)
 
     def test_md4_copy(self):
diff --git a/passlib/tests/test_utils_handlers.py b/passlib/tests/test_utils_handlers.py
index 838d016..9028df4 100644
--- a/passlib/tests/test_utils_handlers.py
+++ b/passlib/tests/test_utils_handlers.py
@@ -13,9 +13,9 @@ import warnings
 from passlib.hash import ldap_md5, sha256_crypt
 from passlib.registry import _unload_handler_name as unload_handler_name, \
     register_crypt_handler, get_crypt_handler
-from passlib.utils import rng, getrandstr, handlers as uh, bytes, b, \
-    to_native_str, to_unicode, MissingBackendError, JYTHON
-from passlib.utils.compat import unicode
+from passlib.utils import getrandstr, JYTHON, rng, to_unicode, MissingBackendError
+from passlib.utils.compat import b, bytes, bascii_to_str, uascii_to_str, unicode
+import passlib.utils.handlers as uh
 from passlib.tests.utils import HandlerCase, TestCase, catch_warnings, \
     dummy_handler_in_registry
 from passlib.utils.compat import u
@@ -44,7 +44,7 @@ class SkeletonTest(TestCase):
                     hash = hash.decode("ascii")
                 if hash not in (u('a'),u('b')):
                     raise ValueError
-                return to_native_str(u('b') if flag else u('a'))
+                return 'b' if flag else 'a'
 
         #check default identify method
         self.assertTrue(d1.identify(u('a')))
@@ -58,7 +58,7 @@ class SkeletonTest(TestCase):
         #check default genconfig method
         self.assertIs(d1.genconfig(), None)
         d1._stub_config = u('b')
-        self.assertEqual(d1.genconfig(), to_native_str('b'))
+        self.assertEqual(d1.genconfig(), 'b')
 
         #check default verify method
         self.assertTrue(d1.verify('s','a'))
@@ -69,9 +69,9 @@ class SkeletonTest(TestCase):
         self.assertRaises(ValueError, d1.verify, 's', 'c')
 
         #check default encrypt method
-        self.assertEqual(d1.encrypt('s'), to_native_str('a'))
-        self.assertEqual(d1.encrypt('s'), to_native_str('a'))
-        self.assertEqual(d1.encrypt('s', flag=True), to_native_str('b'))
+        self.assertEqual(d1.encrypt('s'), 'a')
+        self.assertEqual(d1.encrypt('s'), 'a')
+        self.assertEqual(d1.encrypt('s', flag=True), 'b')
 
     #=========================================================
     #GenericHandler & mixins
@@ -411,7 +411,7 @@ class UnsaltedHash(uh.StaticHandler):
         if isinstance(secret, unicode):
             secret = secret.encode("utf-8")
         data = b("boblious") + secret
-        return to_native_str(hashlib.sha1(data).hexdigest())
+        return bascii_to_str(hashlib.sha1(data).hexdigest())
 
 class SaltedHash(uh.HasSalt, uh.GenericHandler):
     "test algorithm with a salt"
@@ -439,13 +439,13 @@ class SaltedHash(uh.HasSalt, uh.GenericHandler):
 
     def to_string(self):
         hash = u("@salt%s%s") % (self.salt, self.checksum or self._stub_checksum)
-        return to_native_str(hash)
+        return uascii_to_str(hash)
 
     def calc_checksum(self, secret):
         if isinstance(secret, unicode):
             secret = secret.encode("utf-8")
         data = self.salt.encode("ascii") + secret + self.salt.encode("ascii")
-        return to_unicode(hashlib.sha1(data).hexdigest(), "latin-1")
+        return hashlib.sha1(data).hexdigest().decode("ascii")
 
 #=========================================================
 #test sample algorithms - really a self-test of HandlerCase
diff --git a/passlib/tests/utils.py b/passlib/tests/utils.py
index c9a5aff..85c9ea0 100644
--- a/passlib/tests/utils.py
+++ b/passlib/tests/utils.py
@@ -10,7 +10,7 @@ import re
 import os
 import sys
 import tempfile
-from passlib.utils.compat import u, PY27, PY_MIN_32, PY3
+from passlib.utils.compat import PY27, PY_MIN_32, PY3
 
 try:
     import unittest2 as unittest
@@ -30,12 +30,12 @@ if ut_version < 2:
     #used to provide replacement skipTest() method
     from nose.plugins.skip import SkipTest
 #pkg
-from passlib import registry, utils
-from passlib.utils import classproperty, handlers as uh, \
-        has_rounds_info, has_salt_info, MissingBackendError, \
-        rounds_cost_values, b, bytes
-from passlib.utils.compat import iteritems, irange, callable, sb_types, \
-                                 exc_err, unicode
+import passlib.registry as registry
+from passlib.utils import has_rounds_info, has_salt_info, MissingBackendError,\
+                          rounds_cost_values, classproperty
+from passlib.utils.compat import b, bytes, iteritems, irange, callable, \
+                                 sb_types, exc_err, u, unicode
+import passlib.utils.handlers as uh
 #local
 __all__ = [
     #util funcs
@@ -542,7 +542,8 @@ class HandlerCase(TestCase):
                 #this allows use to test as much of the hash's code path
                 #as possible, even if current OS doesn't provide crypt() support
                 #for the hash.
-                self._orig_os_crypt = utils.os_crypt
+                import passlib.utils as mod
+                self._orig_os_crypt = mod.os_crypt
                 def crypt_stub(secret, hash):
                     tmp = h.get_backend()
                     try:
@@ -553,12 +554,13 @@ class HandlerCase(TestCase):
                     if not PY3 and isinstance(hash, unicode):
                         hash = hash.encode("ascii")
                     return hash
-                utils.os_crypt = crypt_stub
+                mod.os_crypt = crypt_stub
             h.set_backend(backend)
 
     def tearDown(self):
         if self._orig_os_crypt:
-            utils.os_crypt = self._orig_os_crypt
+            import passlib.utils as mod
+            mod.os_crypt = self._orig_os_crypt
         if self._orig_backend:
             self.handler.set_backend(self._orig_backend)
 
@@ -1006,7 +1008,8 @@ class HandlerCase(TestCase):
             possible = True
             if handler.has_backend("os_crypt"):
                 def check_crypt(secret, hash):
-                    self.assertEqual(utils.os_crypt(secret, hash), hash,
+                    from passlib.utils import os_crypt
+                    self.assertEqual(os_crypt(secret, hash), hash,
                                      "os_crypt(%r,%r):" % (secret, hash))
                 helpers.append(check_crypt)
 
@@ -1076,7 +1079,8 @@ def _enable_backend_case(handler, backend):
     if enable_option("all-backends") or _is_default_backend(handler, backend):
         if handler.has_backend(backend):
             return True, None
-        if backend == "os_crypt" and utils.safe_os_crypt:
+        from passlib.utils import has_os_crypt
+        if backend == "os_crypt" and has_os_crypt:
             if enable_option("cover") and _has_other_backends(handler, "os_crypt"):
                 #in this case, HandlerCase will monkeypatch os_crypt
                 #to use another backend, just so we can test os_crypt fully.