assorted bugfixes, tweaks, and tests added; based on coverage examination

* test os_crypt backend has functional fallback
* test handler methods accept all unicode/bytes combinations for secret & hash
* fixed some incorrect error messages & types being caught & raised
* other minor cleanups

18 files changed, 453 insertions, 179 deletions

diff --git a/passlib/context.py b/passlib/context.py
index 2eb1a4f..8480ab7 100644
--- a/passlib/context.py
+++ b/passlib/context.py
@@ -20,7 +20,7 @@ from passlib.exc import PasslibConfigWarning, ExpectedStringError
 from passlib.registry import get_crypt_handler, _validate_handler_name
 from passlib.utils import is_crypt_handler, rng, saslprep, tick, to_bytes, \
                           to_unicode
-from passlib.utils.compat import bytes, is_mapping, iteritems, num_types, \
+from passlib.utils.compat import bytes, iteritems, num_types, \
                                  PY3, PY_MIN_32, unicode, SafeConfigParser, \
                                  NativeStringIO, BytesIO, base_string_types
 #pkg
diff --git a/passlib/exc.py b/passlib/exc.py
index 1e78123..1fbe46b 100644
--- a/passlib/exc.py
+++ b/passlib/exc.py
@@ -102,17 +102,25 @@ def _get_name(handler):
 #----------------------------------------------------------------
 # encrypt/verify parameter errors
 #----------------------------------------------------------------
-def ExpectedStringError(value, param):
-    "error message when param was supposed to be unicode or bytes"
-    # NOTE: value is never displayed, since it may sometimes be a password.
+def type_name(value):
+    "return pretty-printed string containing name of value's type"
     cls = value.__class__
     if cls.__module__ and cls.__module__ not in ["__builtin__", "builtins"]:
-        name = "%s.%s" % (cls.__module__, cls.__name__)
+        return "%s.%s" % (cls.__module__, cls.__name__)
     elif value is None:
-        name = 'None'
+        return 'None'
     else:
-        name = cls.__name__
-    return TypeError("%s must be unicode or bytes, not %s" % (param, name))
+        return cls.__name__
+
+def ExpectedTypeError(value, expected, param):
+    "error message when param was supposed to be one type, but found another"
+    # NOTE: value is never displayed, since it may sometimes be a password.
+    name = type_name(value)
+    return TypeError("%s must be %s, not %s" % (param, expected, name))
+
+def ExpectedStringError(value, param):
+    "error message when param was supposed to be unicode or bytes"
+    return ExpectedTypeError(value, "unicode or bytes", param)
 
 def MissingDigestError(handler=None):
     "raised when verify() method gets passed config string instead of hash"
diff --git a/passlib/handlers/bcrypt.py b/passlib/handlers/bcrypt.py
index f07a194..89680a2 100644
--- a/passlib/handlers/bcrypt.py
+++ b/passlib/handlers/bcrypt.py
@@ -186,8 +186,7 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
 
     def _norm_salt(self, salt, **kwds):
         salt = super(bcrypt, self)._norm_salt(salt, **kwds)
-        if not salt:
-            return None
+        assert salt is not None, "HasSalt didn't generate new salt!"
         changed, salt = bcrypt64.check_repair_unused(salt)
         if changed:
             warn(
diff --git a/passlib/handlers/cisco.py b/passlib/handlers/cisco.py
index 184134e..c61a105 100644
--- a/passlib/handlers/cisco.py
+++ b/passlib/handlers/cisco.py
@@ -149,7 +149,7 @@ class cisco_type7(uh.GenericHandler):
             else:
                 raise TypeError("no salt specified")
         if not isinstance(salt, int):
-            raise TypeError("salt must be an integer")
+            raise uh.exc.ExpectedTypeError(salt, "integer", "salt")
         if salt < 0 or salt > self.max_salt_value:
             msg = "salt/offset must be in 0..52 range"
             if self.relaxed:
diff --git a/passlib/handlers/fshp.py b/passlib/handlers/fshp.py
index 3404bd8..2e536ce 100644
--- a/passlib/handlers/fshp.py
+++ b/passlib/handlers/fshp.py
@@ -116,7 +116,7 @@ class fshp(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler):
         if not isinstance(variant, int):
             raise TypeError("fshp variant must be int or known alias")
         if variant not in self._variant_info:
-            raise TypeError("unknown fshp variant")
+            raise ValueError("invalid fshp variant")
         return variant
 
     @property
@@ -152,7 +152,7 @@ class fshp(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler):
         rounds = int(rounds)
         try:
             data = b64decode(data.encode("ascii"))
-        except ValueError:
+        except TypeError:
             raise uh.exc.MalformedHashError(cls)
         salt = data[:salt_size]
         chk = data[salt_size:]
diff --git a/passlib/handlers/md5_crypt.py b/passlib/handlers/md5_crypt.py
index 9441bbc..c963155 100644
--- a/passlib/handlers/md5_crypt.py
+++ b/passlib/handlers/md5_crypt.py
@@ -68,14 +68,13 @@ def _raw_md5_crypt(pwd, salt, use_apr=False):
     #=====================================================================
 
     #validate secret
+    # XXX: not sure what official unicode policy is, using this as default
     if isinstance(pwd, unicode):
-        # XXX: not sure what official unicode policy is, using this as default
         pwd = pwd.encode("utf-8")
-    elif not isinstance(pwd, bytes):
-        raise TypeError("password must be bytes or unicode")
+    assert isinstance(pwd, bytes), "pwd not unicode or bytes"
     pwd_len = len(pwd)
 
-    #validate salt
+    #validate salt - should have been taken care of by caller
     assert isinstance(salt, unicode), "salt not unicode"
     salt = salt.encode("ascii")
     assert len(salt) < 9, "salt too large"
diff --git a/passlib/handlers/misc.py b/passlib/handlers/misc.py
index cb812ff..7121707 100644
--- a/passlib/handlers/misc.py
+++ b/passlib/handlers/misc.py
@@ -141,6 +141,8 @@ class unix_disabled(object):
             # NOTE: config/hash will generally be "!" or "*",
             # but we want to preserve it in case it has some other content,
             # such as ``"!"  + original hash``, which glibc uses.
+            # XXX: should this detect mcf header, or other things re:
+            # local system policy?
             return to_native_str(config, errname="config")
         else:
             return to_native_str(marker or cls.marker, errname="marker")
diff --git a/passlib/handlers/scram.py b/passlib/handlers/scram.py
index e7d6399..036d7c2 100644
--- a/passlib/handlers/scram.py
+++ b/passlib/handlers/scram.py
@@ -285,7 +285,7 @@ class scram(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler):
                 raise ValueError("SCRAM limits algorithm names to "
                                  "9 characters: %r" % (alg,))
             if not isinstance(digest, bytes):
-                raise TypeError("digests must be raw bytes")
+                raise uh.exc.ExpectedTypeError(digest, "raw bytes", "digests")
             # TODO: verify digest size (if digest is known)
         if 'sha-1' not in checksum:
             # NOTE: required because of SCRAM spec.
@@ -384,7 +384,8 @@ class scram(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler):
                 if alg in chkmap:
                     other = self._calc_checksum(secret, alg)
                     return consteq(other, chkmap[alg])
-            # there should *always* be at least sha-1.
+            # there should always be sha-1 at the very least,
+            # or something went wrong inside _norm_algs()
             raise AssertionError("sha-1 digest not found!")
 
     #=========================================================
diff --git a/passlib/hosts.py b/passlib/hosts.py
index 5d3abc6..5836c98 100644
--- a/passlib/hosts.py
+++ b/passlib/hosts.py
@@ -79,7 +79,7 @@ if has_crypt:
             #only offer fallback if there's another scheme in front,
             #as this can't actually hash any passwords
             yield "unix_disabled"
-        else:
+        else: # pragma: no cover
             #no idea what OS this could happen on, but just in case...
             warn("crypt.crypt() function is present, but doesn't support any "
                  "formats known to passlib!", PasslibRuntimeWarning)
diff --git a/passlib/tests/test_ext_django.py b/passlib/tests/test_ext_django.py
index 890c87b..f4ea932 100644
--- a/passlib/tests/test_ext_django.py
+++ b/passlib/tests/test_ext_django.py
@@ -430,85 +430,70 @@ class PluginTest(TestCase):
     descriptionPrefix = "passlib.ext.django plugin"
 
     def setUp(self):
-        #remove django patch
+        super(PluginTest, self).setUp()
+
+        # remove django patch now, and at end
         utils.set_django_password_context(None)
+        self.addCleanup(utils.set_django_password_context, None)
 
-        #ensure django settings are empty
+        # ensure django settings are empty
         update_settings(
             PASSLIB_CONTEXT=_NOTSET,
             PASSLIB_GET_CATEGORY=_NOTSET,
         )
 
-        #unload module so it's re-run
+        # unload module so it's re-run when imported
         sys.modules.pop("passlib.ext.django.models", None)
 
-    def tearDown(self):
-        #remove django patch
-        utils.set_django_password_context(None)
+    def check_hashes(self, tests, default_scheme, deprecated=[], load=True):
+        """run through django api to verify patch is configured & functioning"""
+        # load extension if it hasn't been already.
+        if load:
+            import passlib.ext.django.models
 
-    def check_hashes(self, tests, new_hash=None, deprecated=None):
-        u = FakeUser()
-        deprecated = None
+        # create fake user object
+        user = FakeUser()
 
-        # check new hash construction
-        if new_hash:
-            u.set_password("placeholder")
-            handler = get_crypt_handler(new_hash)
-            self.assertTrue(handler.identify(u.password))
+        # check new hashes constructed using default scheme
+        user.set_password("stub")
+        handler = get_crypt_handler(default_scheme)
+        self.assertTrue(handler.identify(user.password),
+                        "handler failed to identify hash: %r %r" %
+                        (default_scheme, user.password))
 
         # run against hashes from tests...
         for test in tests:
             for secret, hash in test.iter_known_hashes():
 
                 # check against valid password
-                u.password = hash
+                user.password = hash
                 if has_django0 and isinstance(secret, unicode):
                     secret = secret.encode("utf-8")
-                self.assertTrue(u.check_password(secret))
-                if new_hash and deprecated and test.handler.name in deprecated:
+                self.assertTrue(user.check_password(secret))
+                if deprecated and test.handler.name in deprecated:
                     self.assertFalse(handler.identify(hash))
-                    self.assertTrue(handler.identify(u.password))
+                    self.assertTrue(handler.identify(user.password))
 
                 # check against invalid password
-                u.password = hash
-                self.assertFalse(u.check_password('x'+secret))
-                if new_hash and deprecated and test.handler.name in deprecated:
+                user.password = hash
+                self.assertFalse(user.check_password('x'+secret))
+                if deprecated and test.handler.name in deprecated:
                     self.assertFalse(handler.identify(hash))
-                    self.assertEqual(u.password, hash)
+                    self.assertEqual(user.password, hash)
 
         # check disabled handling
         if has_django1:
-            u.set_password(None)
+            user.set_password(None)
             handler = get_crypt_handler("django_disabled")
-            self.assertTrue(handler.identify(u.password))
-            self.assertFalse(u.check_password('placeholder'))
+            self.assertTrue(handler.identify(user.password))
+            self.assertFalse(user.check_password('placeholder'))
 
-    def test_00_actual_django(self):
-        "test actual Django behavior has not changed"
-        #NOTE: if this test fails,
-        #      probably means newer version of Django,
-        #      and passlib's policies should be updated.
+    def check_django_stock(self, load=True):
         self.check_hashes(django_hash_tests,
                           "django_salted_sha1",
-                          ["hex_md5"])
-
-    def test_01_explicit_unset(self, value=None):
-        "test PASSLIB_CONTEXT = None"
-        update_settings(
-            PASSLIB_CONTEXT=value,
-        )
-        import passlib.ext.django.models
-        self.check_hashes(django_hash_tests,
-                          "django_salted_sha1",
-                          ["hex_md5"])
-
-    def test_02_stock_ctx(self):
-        "test PASSLIB_CONTEXT = utils.STOCK_CTX"
-        self.test_01_explicit_unset(value=utils.STOCK_CTX)
+                          ["hex_md5"], load=load)
 
-    def test_03_implicit_default_ctx(self):
-        "test PASSLIB_CONTEXT unset"
-        import passlib.ext.django.models
+    def check_passlib_stock(self):
         self.check_hashes(default_hash_tests,
                           "sha512_crypt",
                           ["hex_md5", "django_salted_sha1",
@@ -516,21 +501,43 @@ class PluginTest(TestCase):
                            "django_des_crypt",
                            ])
 
-    def test_04_explicit_default_ctx(self):
+    def test_10_django(self):
+        "test actual Django behavior has not changed"
+        #NOTE: if this test fails,
+        #      probably means newer version of Django,
+        #      and passlib's policies should be updated.
+        self.check_django_stock(load=False)
+
+    def test_11_none(self):
+        "test PASSLIB_CONTEXT=None"
+        update_settings(PASSLIB_CONTEXT=None)
+        self.check_django_stock(load=False)
+
+    def test_12_string(self):
+        "test PASSLIB_CONTEXT=string"
+        update_settings(PASSLIB_CONTEXT=utils.STOCK_CTX)
+        self.check_django_stock(load=False)
+
+    def test_13_unset(self):
+        "test unset PASSLIB_CONTEXT uses default"
+        self.check_passlib_stock()
+
+    def test_14_default(self):
         "test PASSLIB_CONTEXT = utils.DEFAULT_CTX"
-        update_settings(
-            PASSLIB_CONTEXT=utils.DEFAULT_CTX,
-        )
-        self.test_03_implicit_default_ctx()
+        update_settings(PASSLIB_CONTEXT=utils.DEFAULT_CTX)
+        self.check_passlib_stock()
 
-    def test_05_default_ctx_alias(self):
+    def test_15_default_alias(self):
         "test PASSLIB_CONTEXT = 'passlib-default'"
-        update_settings(
-            PASSLIB_CONTEXT="passlib-default",
-        )
-        self.test_03_implicit_default_ctx()
+        update_settings(PASSLIB_CONTEXT="passlib-default")
+        self.check_passlib_stock()
+
+    def test_16_invalid(self):
+        "test PASSLIB_CONTEXT = invalid type"
+        update_settings(PASSLIB_CONTEXT=123)
+        self.assertRaises(TypeError, __import__, 'passlib.ext.django.models')
 
-    def test_06_categories(self):
+    def test_20_categories(self):
         "test PASSLIB_GET_CATEGORY unset"
         update_settings(
             PASSLIB_CONTEXT=category_context.policy.to_string(),
@@ -541,7 +548,7 @@ class PluginTest(TestCase):
         self.assertEqual(get_cc_rounds(is_staff=True), 2000)
         self.assertEqual(get_cc_rounds(is_superuser=True), 3000)
 
-    def test_07_categories_explicit(self):
+    def test_21_categories_explicit(self):
         "test PASSLIB_GET_CATEGORY = function"
         def get_category(user):
             return user.first_name or None
@@ -556,7 +563,7 @@ class PluginTest(TestCase):
         self.assertEqual(get_cc_rounds(first_name='staff'), 2000)
         self.assertEqual(get_cc_rounds(first_name='superuser'), 3000)
 
-    def test_08_categories_disabled(self):
+    def test_22_categories_disabled(self):
         "test PASSLIB_GET_CATEGORY = None"
         update_settings(
             PASSLIB_CONTEXT = category_context.policy.to_string(),
diff --git a/passlib/tests/test_handlers.py b/passlib/tests/test_handlers.py
index 63e457c..d1c5cbf 100644
--- a/passlib/tests/test_handlers.py
+++ b/passlib/tests/test_handlers.py
@@ -497,6 +497,7 @@ class cisco_pix_test(UserHandlerMixin, HandlerCase):
 class cisco_type7_test(HandlerCase):
     handler = hash.cisco_type7
     salt_bits = 4
+    salt_type = int
 
     known_correct_hashes = [
         #
@@ -539,6 +540,41 @@ class cisco_type7_test(HandlerCase):
         (UPASS_TABLE, '0958EDC8A9F495F6F8A5FD'),
     ]
 
+    known_unidentified_hashes = [
+        # salt with hex value
+        "0A480E051A33490E",
+
+        # salt value > 52. this may in fact be valid, but we reject it for now
+        # (see docs for more).
+        '99400E4812',
+    ]
+
+    def test_90_decode(self):
+        "test cisco_type7.decode()"
+        from passlib.utils import to_unicode, to_bytes
+
+        handler = self.handler
+        for secret, hash in self.known_correct_hashes:
+            usecret = to_unicode(secret)
+            bsecret = to_bytes(secret)
+            self.assertEqual(handler.decode(hash), usecret)
+            self.assertEqual(handler.decode(hash, None), bsecret)
+
+        self.assertRaises(UnicodeDecodeError, handler.decode,
+                          '0958EDC8A9F495F6F8A5FD', 'ascii')
+
+    def test_91_salt(self):
+        "test salt value border cases"
+        handler = self.handler
+        self.assertRaises(TypeError, handler, salt=None)
+        handler(salt=None, use_defaults=True)
+        self.assertRaises(TypeError, handler, salt='abc')
+        self.assertRaises(ValueError, handler, salt=-10)
+        with catch_warnings(record=True) as wlog:
+            h = handler(salt=100, relaxed=True)
+            self.consumeWarningList(wlog, ["salt/offset must be.*"])
+            self.assertEqual(h.salt, 52)
+
 #=========================================================
 # crypt16
 #=========================================================
@@ -794,6 +830,9 @@ class fshp_test(HandlerCase):
         ]
 
     known_malformed_hashes = [
+        # bad base64 padding
+        '{FSHP0|0|1}qUqP5cyxm6YcTAhz05Hph5gvu9M',
+
         # wrong salt size
         '{FSHP0|1|1}qUqP5cyxm6YcTAhz05Hph5gvu9M=',
 
@@ -801,6 +840,32 @@ class fshp_test(HandlerCase):
         '{FSHP0|0|A}qUqP5cyxm6YcTAhz05Hph5gvu9M=',
     ]
 
+    def test_90_variant(self):
+        "test variant keyword"
+        handler = self.handler
+        kwds = dict(salt='a', rounds=1)
+
+        # accepts ints
+        handler(variant=1, **kwds)
+
+        # accepts bytes or unicode
+        handler(variant=u('1'), **kwds)
+        handler(variant=b('1'), **kwds)
+
+        # aliases
+        handler(variant=u('sha256'), **kwds)
+        handler(variant=b('sha256'), **kwds)
+
+        # rejects None
+        self.assertRaises(TypeError, handler, variant=None, **kwds)
+
+        # rejects other types
+        self.assertRaises(TypeError, handler, variant=complex(1,1), **kwds)
+
+        # invalid variant
+        self.assertRaises(ValueError, handler, variant='9', **kwds)
+        self.assertRaises(ValueError, handler, variant=9, **kwds)
+
 #=========================================================
 #hex digests
 #=========================================================
@@ -1080,6 +1145,9 @@ class _md5_crypt_test(HandlerCase):
     known_malformed_hashes = [
         # bad char in otherwise correct hash \/
            '$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o!',
+
+        # too many fields
+        '$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o.$',
         ]
 
     platform_crypt_support = dict(
@@ -2005,6 +2073,12 @@ class _sha1_crypt_test(HandlerCase):
 
         # zero padded rounds
         '$sha1$01773$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH',
+
+        # too many fields
+        '$sha1$21773$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH$',
+
+        # empty rounds field
+        '$sha1$$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH$',
     ]
 
     platform_crypt_support = dict(
@@ -2316,6 +2390,14 @@ class sun_md5_crypt_test(HandlerCase):
     ]
 
     known_malformed_hashes = [
+        # unexpected end of hash
+        "$md5,rounds=5000",
+
+        # bad rounds
+        "$md5,rounds=500A$xxxx",
+        "$md5,rounds=0500$xxxx",
+        "$md5,rounds=0$xxxx",
+
         # bad char in otherwise correct hash
         "$md5$RPgL!6IJ$WTvAlUJ7MqH5xak2FMEwS/",
 
@@ -2369,6 +2451,16 @@ class unix_disabled_test(HandlerCase):
     # TODO: test custom marker support
     # TODO: test default marker selection
 
+    def test_90_preserves_existing(self):
+        "test preserves existing disabled hash"
+        handler = self.handler
+
+        # use marker if no hash
+        self.assertEqual(handler.genhash("stub", None), handler.marker)
+
+        # use hash if provided and valid
+        self.assertEqual(handler.genhash("stub", "!asd"), "!asd")
+
 class unix_fallback_test(HandlerCase):
     handler = hash.unix_fallback
     accepts_all_hashes = True
diff --git a/passlib/tests/test_registry.py b/passlib/tests/test_registry.py
index 1990919..3f18271 100644
--- a/passlib/tests/test_registry.py
+++ b/passlib/tests/test_registry.py
@@ -126,6 +126,8 @@ class RegistryTest(TestCase):
         self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name=None)))
         self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="AB_CD")))
         self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="ab-cd")))
+        self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="ab__cd")))
+        self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="default")))
 
         class dummy_1(uh.StaticHandler):
             name = "dummy_1"
diff --git a/passlib/tests/test_utils.py b/passlib/tests/test_utils.py
index 4c3a6d2..4f6b62d 100644
--- a/passlib/tests/test_utils.py
+++ b/passlib/tests/test_utils.py
@@ -26,6 +26,60 @@ class MiscTest(TestCase):
 
     #NOTE: could test xor_bytes(), but it's exercised well enough by pbkdf2 test
 
+    def test_classproperty(self):
+        from passlib.utils import classproperty
+
+        class test(object):
+            xvar = 1
+            @classproperty
+            def xprop(cls):
+                return cls.xvar
+
+        self.assertEqual(test.xprop, 1)
+        prop = test.__dict__['xprop']
+        self.assertIs(prop.im_func, prop.__func__)
+
+    def test_deprecated_function(self):
+        from passlib.utils import deprecated_function
+        # NOTE: not comprehensive, just tests the basic behavior
+
+        @deprecated_function(deprecated="1.6", removed="1.8")
+        def test_func(*args):
+            "test docstring"
+            return args
+
+        self.assertTrue(".. deprecated::" in test_func.__doc__)
+
+        with catch_warnings(record=True) as wlog:
+            self.assertEqual(test_func(1,2), (1,2))
+            self.consumeWarningList(wlog,[
+                dict(category=DeprecationWarning,
+                     message="the function passlib.tests.test_utils.test_func() "
+                             "is deprecated as of Passlib 1.6, and will be "
+                             "removed in Passlib 1.8."
+                ),
+            ])
+
+    def test_memoized_property(self):
+        from passlib.utils import memoized_property
+
+        class dummy(object):
+            counter = 0
+
+            @memoized_property
+            def value(self):
+                value = self.counter
+                self.counter = value+1
+                return value
+
+        d = dummy()
+        self.assertEqual(d.value, 0)
+        self.assertEqual(d.value, 0)
+        self.assertEqual(d.counter, 1)
+
+        prop = dummy.value
+        self.assertIs(prop.im_func, prop.__func__)
+
     def test_getrandbytes(self):
         "test getrandbytes()"
         from passlib.utils import getrandbytes, rng
@@ -284,6 +338,8 @@ class MiscTest(TestCase):
             # unassigned code points (as of unicode 3.2)
         self.assertRaises(ValueError, sp, u("\u0900"))
         self.assertRaises(ValueError, sp, u("\uFFF8"))
+            # tagging characters
+        self.assertRaises(ValueError, sp, u("\U000e0001"))
 
         # verify bidi behavior
             # if starts with R/AL -- must end with R/AL
@@ -524,6 +580,30 @@ b("""    0000000000000000 0000000000000000 8CA64DE9C1B123A7
 #=========================================================
 # base64engine
 #=========================================================
+class Base64EngineTest(TestCase):
+    "test standalone parts of Base64Engine"
+    # NOTE: most Base64Engine testing done via _Base64Test subclasses below.
+
+    def test_constructor(self):
+        from passlib.utils import Base64Engine, AB64_CHARS
+
+        # bad charmap type
+        self.assertRaises(TypeError, Base64Engine, 1)
+
+        # bad charmap size
+        self.assertRaises(ValueError, Base64Engine, AB64_CHARS[:-1])
+
+        # dup charmap letter
+        self.assertRaises(ValueError, Base64Engine, AB64_CHARS[:-1] + "A")
+
+    def test_ab64(self):
+        from passlib.utils import ab64_decode
+        # TODO: make ab64_decode (and a b64 variant) *much* stricter about
+        # padding chars, etc.
+
+        # 1 mod 4 not valid
+        self.assertRaises(ValueError, ab64_decode, "abcde")
+
 class _Base64Test(TestCase):
     "common tests for all Base64Engine instances"
     #=========================================================
@@ -724,6 +804,8 @@ class _Base64Test(TestCase):
             out = engine.decode_bytes(tmp)
             self.assertEqual(out, result)
 
+        self.assertRaises(TypeError, engine.encode_transposed_bytes, u("a"), [])
+
     def test_decode_transposed_bytes(self):
         "test decode_transposed_bytes()"
         engine = self.engine
diff --git a/passlib/tests/utils.py b/passlib/tests/utils.py
index 713824e..a75f428 100644
--- a/passlib/tests/utils.py
+++ b/passlib/tests/utils.py
@@ -42,7 +42,7 @@ from passlib.utils import has_rounds_info, has_salt_info, rounds_cost_values, \
                           classproperty, rng, getrandstr, is_ascii_safe, to_native_str, \
                           repeat_string
 from passlib.utils.compat import b, bytes, iteritems, irange, callable, \
-                                 base_string_types, exc_err, u, unicode
+                                 base_string_types, exc_err, u, unicode, PY2
 import passlib.utils.handlers as uh
 #local
 __all__ = [
@@ -134,6 +134,18 @@ def get_file(path):
     with open(path, "rb") as fh:
         return fh.read()
 
+def tonn(source):
+    "convert native string to non-native string"
+    if not isinstance(source, str):
+        return source
+    elif PY3:
+        return source.encode("utf-8")
+    else:
+        try:
+            return source.decode("utf-8")
+        except UnicodeDecodeError:
+            return source.decode("latin-1")
+
 #=========================================================
 #custom test base
 #=========================================================
@@ -741,7 +753,7 @@ class HandlerCase(TestCase):
 
         # XXX: any more checks needed?
 
-    def test_02_config(self):
+    def test_02_config_workflow(self):
         """test basic config-string workflow
 
         this tests that genconfig() returns the expected types,
@@ -785,7 +797,7 @@ class HandlerCase(TestCase):
         else:
             self.assertRaises(TypeError, self.do_identify, config)
 
-    def test_03_hash(self):
+    def test_03_hash_workflow(self):
         """test basic hash-string workflow.
 
         this tests that encrypt()'s hashes are accepted
@@ -835,8 +847,36 @@ class HandlerCase(TestCase):
             #
             self.assertTrue(self.do_identify(result))
 
+    def test_04_hash_types(self):
+        "test hashes can be unicode or bytes"
+        # this runs through workflow similar to 03, but wraps
+        # everything using tonn() so we test unicode under py2,
+        # and bytes under py3.
+
+        # encrypt using non-native secret
+        result = self.do_encrypt(tonn('stub'))
+        self.check_returned_native_str(result, "encrypt")
+
+        # verify using non-native hash
+        self.check_verify('stub', tonn(result))
+
+        # verify using non-native hash AND secret
+        self.check_verify(tonn('stub'), tonn(result))
+
+        # genhash using non-native hash
+        other = self.do_genhash('stub', tonn(result))
+        self.check_returned_native_str(other, "genhash")
+        self.assertEqual(other, result)
 
-    def test_04_backends(self):
+        # genhash using non-native hash AND secret
+        other = self.do_genhash(tonn('stub'), tonn(result))
+        self.check_returned_native_str(other, "genhash")
+        self.assertEqual(other, result)
+
+        # identify using non-native hash
+        self.assertTrue(self.do_identify(tonn(result)))
+
+    def test_05_backends(self):
         "test multi-backend support"
         handler = self.handler
         if not hasattr(handler, "set_backend"):
@@ -1065,6 +1105,34 @@ class HandlerCase(TestCase):
                 self.assertRaises(ValueError, self.do_genconfig, salt=c*chunk,
                                   __msg__="invalid salt char %r:" % (c,))
 
+    @property
+    def salt_type(self):
+        "hack to determine salt keyword's datatype"
+        # NOTE: cisco_type7 uses 'int'
+        if getattr(self.handler, "_salt_is_bytes", False):
+            return bytes
+        else:
+            return unicode
+
+    def test_15_salt_type(self):
+        "test non-string salt values"
+        self.require_salt()
+        salt_type = self.salt_type
+
+        # should always throw error for random class.
+        class fake(object):
+            pass
+        self.assertRaises(TypeError, self.do_encrypt, 'stub', salt=fake())
+
+        # unicode should be accepted only if salt_type is unicode.
+        if salt_type is not unicode:
+            self.assertRaises(TypeError, self.do_encrypt, 'stub', salt=u('x'))
+
+        # bytes should be accepted only if salt_type is bytes,
+        # OR if salt type is unicode and running PY2 - to allow native strings.
+        if not (salt_type is bytes or (PY2 and salt_type is unicode)):
+            self.assertRaises(TypeError, self.do_encrypt, 'stub', salt=b('x'))
+
     #==============================================================
     # rounds
     #==============================================================
@@ -1672,11 +1740,8 @@ class HandlerCase(TestCase):
                 return rng.choice(handler.ident_values)
 
     #=========================================================
-    # test 8x - mixin tests
-    # test 9x - handler-specific tests
-    #=========================================================
-
-    #=========================================================
+    #       test 8x - mixin tests
+    #       test 9x - handler-specific tests
     # eoc
     #=========================================================
 
@@ -1746,20 +1811,27 @@ class OsCryptMixin(HandlerCase):
     #=========================================================
     # custom tests
     #=========================================================
-    def test_80_faulty_crypt(self):
-        "test with faulty crypt()"
-        # patch safe_crypt to return mock value.
+    def _use_mock_crypt(self):
+        "patch safe_crypt() so it returns mock value"
         import passlib.utils as mod
-        self.addCleanup(setattr, mod, "_crypt", mod._crypt)
+        if not self.using_patched_crypt:
+            self.addCleanup(setattr, mod, "_crypt", mod._crypt)
         crypt_value = [None]
         mod._crypt = lambda secret, config: crypt_value[0]
+        def setter(value):
+            crypt_value[0] = value
+        return setter
 
-        # prepare framework
+    def test_80_faulty_crypt(self):
+        "test with faulty crypt()"
         hash = self.get_sample_hash()[1]
         exc_types = (AssertionError,)
+        setter = self._use_mock_crypt()
 
         def test(value):
-            crypt_value[0] = value
+            # set safe_crypt() to return specified value, and
+            # make sure assertion error is raised by handler.
+            setter(value)
             self.assertRaises(exc_types, self.do_genhash, "stub", hash)
             self.assertRaises(exc_types, self.do_encrypt, "stub")
             self.assertRaises(exc_types, self.do_verify, "stub", hash)
@@ -1768,7 +1840,26 @@ class OsCryptMixin(HandlerCase):
         test(hash[:-1]) # detect too short
         test(hash + 'x') # detect too long
 
-    def test_81_crypt_support(self):
+    def test_81_crypt_fallback(self):
+        "test per-call crypt() fallback"
+        # set safe_crypt to return None
+        setter = self._use_mock_crypt()
+        setter(None)
+        if _find_alternate_backend(self.handler, "os_crypt"):
+            # handler should have a fallback to use
+            h1 = self.do_encrypt("stub")
+            h2 = self.do_genhash("stub", h1)
+            self.assertEqual(h2, h1)
+            self.assertTrue(self.do_verify("stub", h1))
+        else:
+            # handler should give up
+            from passlib.exc import MissingBackendError
+            hash = self.get_sample_hash()[1]
+            self.assertRaises(MissingBackendError, self.do_encrypt, 'stub')
+            self.assertRaises(MissingBackendError, self.do_genhash, 'stub', hash)
+            self.assertRaises(MissingBackendError, self.do_verify, 'stub', hash)
+
+    def test_82_crypt_support(self):
         "test crypt support detection"
         platform = sys.platform
         for name, flag in self.platform_crypt_support.items():
diff --git a/passlib/utils/__init__.py b/passlib/utils/__init__.py
index e0b9e75..9eb8eab 100644
--- a/passlib/utils/__init__.py
+++ b/passlib/utils/__init__.py
@@ -162,50 +162,50 @@ def deprecated_function(msg=None, deprecated=None, removed=None, updoc=True):
         return wrapper
     return build
 
-def relocated_function(target, msg=None, name=None, deprecated=None, mod=None,
-                       removed=None, updoc=True):
-    """constructor to create alias for relocated function.
-
-    :arg target: import path to target
-    :arg msg: optional msg, default chosen if omitted
-    :kwd deprecated: release where function was first deprecated
-    :kwd removed: release where function will be removed
-    :kwd updoc: add notice to docstring (default ``True``)
-    """
-    target_mod, target_name = target.rsplit(".",1)
-    if mod is None:
-        import inspect
-        mod = inspect.currentframe(1).f_globals["__name__"]
-    if not name:
-        name = target_name
-    if msg is None:
-        msg = ("the function %(mod)s.%(name)s() has been moved to "
-               "%(target_mod)s.%(target_name)s(), the old location is deprecated")
-        if deprecated:
-            msg += " as of Passlib %(deprecated)s"
-        if removed:
-            msg += ", and will be removed in Passlib %(removed)s"
-        msg += "."
-    msg %= dict(
-        mod=mod,
-        name=name,
-        target_mod=target_mod,
-        target_name=target_name,
-        deprecated=deprecated,
-        removed=removed,
-    )
-    state = [None]
-    def wrapper(*args, **kwds):
-        warn(msg, DeprecationWarning, stacklevel=2)
-        func = state[0]
-        if func is None:
-            module = __import__(target_mod, fromlist=[target_name], level=0)
-            func = state[0] = getattr(module, target_name)
-        return func(*args, **kwds)
-    wrapper.__module__ = mod
-    wrapper.__name__ = name
-    wrapper.__doc__ = msg
-    return wrapper
+##def relocated_function(target, msg=None, name=None, deprecated=None, mod=None,
+##                       removed=None, updoc=True):
+##    """constructor to create alias for relocated function.
+##
+##    :arg target: import path to target
+##    :arg msg: optional msg, default chosen if omitted
+##    :kwd deprecated: release where function was first deprecated
+##    :kwd removed: release where function will be removed
+##    :kwd updoc: add notice to docstring (default ``True``)
+##    """
+##    target_mod, target_name = target.rsplit(".",1)
+##    if mod is None:
+##        import inspect
+##        mod = inspect.currentframe(1).f_globals["__name__"]
+##    if not name:
+##        name = target_name
+##    if msg is None:
+##        msg = ("the function %(mod)s.%(name)s() has been moved to "
+##               "%(target_mod)s.%(target_name)s(), the old location is deprecated")
+##        if deprecated:
+##            msg += " as of Passlib %(deprecated)s"
+##        if removed:
+##            msg += ", and will be removed in Passlib %(removed)s"
+##        msg += "."
+##    msg %= dict(
+##        mod=mod,
+##        name=name,
+##        target_mod=target_mod,
+##        target_name=target_name,
+##        deprecated=deprecated,
+##        removed=removed,
+##    )
+##    state = [None]
+##    def wrapper(*args, **kwds):
+##        warn(msg, DeprecationWarning, stacklevel=2)
+##        func = state[0]
+##        if func is None:
+##            module = __import__(target_mod, fromlist=[target_name], level=0)
+##            func = state[0] = getattr(module, target_name)
+##        return func(*args, **kwds)
+##    wrapper.__module__ = mod
+##    wrapper.__name__ = name
+##    wrapper.__doc__ = msg
+##    return wrapper
 
 class memoized_property(object):
     """decorator which invokes method once, then replaces attr with result"""
@@ -309,7 +309,7 @@ def consteq(left, right):
     return result == 0
 
 @deprecated_function(deprecated="1.6", removed="1.8")
-def splitcomma(source, sep=","):
+def splitcomma(source, sep=","): # pragma: no cover
     """split comma-separated string into list of elements,
     stripping whitespace and discarding empty elements.
     """
@@ -465,7 +465,7 @@ def render_bytes(source, *args):
 # NOTE: deprecating bytes<->int in favor of just using struct module.
 
 @deprecated_function(deprecated="1.6", removed="1.8")
-def bytes_to_int(value):
+def bytes_to_int(value): # pragma: no cover
     "decode string of bytes as single big-endian integer"
     from passlib.utils.compat import byte_elem_value
     out = 0
@@ -474,7 +474,7 @@ def bytes_to_int(value):
     return out
 
 @deprecated_function(deprecated="1.6", removed="1.8")
-def int_to_bytes(value, count):
+def int_to_bytes(value, count): # pragma: no cover
     "encodes integer into single big-endian byte string"
     assert value < (1<<(8*count)), "value too large for %d bytes: %d" % (count, value)
     return join_byte_values(
@@ -577,10 +577,10 @@ def to_unicode(source, source_encoding="utf-8", errname="value"):
         * returns unicode strings unchanged.
         * returns bytes strings decoded using *source_encoding*
     """
+    assert source_encoding
     if isinstance(source, unicode):
         return source
     elif isinstance(source, bytes):
-        assert source_encoding
         return source.decode(source_encoding)
     else:
         raise ExpectedStringError(source, errname)
@@ -624,7 +624,7 @@ add_doc(to_native_str,
     """)
 
 @deprecated_function(deprecated="1.6", removed="1.7")
-def to_hash_str(source, encoding="ascii"):
+def to_hash_str(source, encoding="ascii"): # pragma: no cover
     "deprecated, use to_native_str() instead"
     return to_native_str(source, encoding, errname="hash")
 
@@ -698,7 +698,7 @@ class Base64Engine(object):
         if isinstance(charmap, unicode):
             charmap = charmap.encode("latin-1")
         elif not isinstance(charmap, bytes):
-            raise TypeError("charmap must be unicode/bytes string")
+            raise ExpectedStringError(charmap, "charmap")
         if len(charmap) != 64:
             raise ValueError("charmap must be 64 characters in length")
         if len(set(charmap)) != 64:
@@ -1145,8 +1145,7 @@ class Base64Engine(object):
         :returns:
             a string of length ``int(ceil(bits/6.0))``.
         """
-        if value < 0:
-            raise ValueError("value cannot be negative")
+        assert value >= 0, "caller did not sanitize input"
         pad = -bits % 6
         bits += pad
         if self.big:
@@ -1298,22 +1297,14 @@ else:
             if isinstance(secret, bytes):
                 # Python 3's crypt() only accepts unicode, which is then
                 # encoding using utf-8 before passing to the C-level crypt().
-                # so we have to decode the secret, but also check that it
-                # re-encodes to the original sequence of bytes... otherwise
-                # the call to crypt() will digest the wrong value.
+                # so we have to decode the secret.
                 orig = secret
                 try:
                     secret = secret.decode("utf-8")
                 except UnicodeDecodeError:
                     return None
-                if secret.encode("utf-8") != orig:
-                    # just in case original encoding wouldn't be reproduced
-                    # during call to os_crypt. not sure if/how this could
-                    # happen, but being paranoid.
-                    from passlib.exc import PasslibRuntimeWarning
-                    warn("utf-8 password didn't re-encode correctly!",
-                         PasslibRuntimeWarning)
-                    return None
+                assert secret.encode("utf-8") == orig, \
+                            "utf-8 spec says this can't happen!"
             if _NULL in secret:
                 raise ValueError("null character in secret")
             if isinstance(hash, bytes):
diff --git a/passlib/utils/compat.py b/passlib/utils/compat.py
index 7bffb15..77f9c3c 100644
--- a/passlib/utils/compat.py
+++ b/passlib/utils/compat.py
@@ -34,7 +34,7 @@ __all__ = [
     'print_',
 
     # type detection
-    'is_mapping',
+##    'is_mapping',
     'callable',
     'int_types',
     'num_types',
@@ -234,9 +234,9 @@ else:
 #=============================================================================
 # typing
 #=============================================================================
-def is_mapping(obj):
-    # non-exhaustive check, enough to distinguish from lists, etc
-    return hasattr(obj, "items")
+##def is_mapping(obj):
+##    # non-exhaustive check, enough to distinguish from lists, etc
+##    return hasattr(obj, "items")
 
 if (3,0) <= sys.version_info < (3,2):
     # callable isn't dead, it's just resting
@@ -390,7 +390,7 @@ class _LazyOverlayModule(ModuleType):
         attrs.update(self.__dict__)
         attrs.update(self.__attrmap)
         proxy = self.__proxy
-        if proxy:
+        if proxy is not None:
             attrs.update(dir(proxy))
         return list(attrs)
 
diff --git a/passlib/utils/handlers.py b/passlib/utils/handlers.py
index fbf7b69..47e83b0 100644
--- a/passlib/utils/handlers.py
+++ b/passlib/utils/handlers.py
@@ -19,11 +19,11 @@ from passlib.exc import MissingBackendError, PasslibConfigWarning, \
 from passlib.registry import get_crypt_handler
 from passlib.utils import classproperty, consteq, getrandstr, getrandbytes,\
                           BASE64_CHARS, HASH64_CHARS, rng, to_native_str, \
-                          is_crypt_handler, deprecated_function, to_unicode, \
+                          is_crypt_handler, to_unicode, \
                           MAX_PASSWORD_SIZE
 from passlib.utils.compat import b, join_byte_values, bytes, irange, u, \
                                  uascii_to_str, join_unicode, unicode, str_to_uascii, \
-                                 join_unicode, base_string_types
+                                 join_unicode, base_string_types, PY2
 # local
 __all__ = [
     # helpers for implementing MCF handlers
@@ -173,7 +173,7 @@ def parse_mc3(hash, prefix, sep=_UDOLLAR, rounds_base=10,
     elif rounds:
         rounds = int(rounds, rounds_base)
     elif default_rounds is None:
-        raise exc.MalformedHashError(handler, "missing rounds field")
+        raise exc.MalformedHashError(handler, "empty rounds field")
     else:
         rounds = default_rounds
 
@@ -411,15 +411,15 @@ class GenericHandler(object):
             # NOTE: no clear route to reasonbly convert unicode -> raw bytes,
             # so relaxed does nothing here
             if not isinstance(checksum, bytes):
-                raise TypeError("checksum must be byte string")
+                raise exc.ExpectedTypeError(checksum, "bytes", "checksum")
 
         elif not isinstance(checksum, unicode):
-            if self.relaxed:
+            if isinstance(checksum, bytes) and self.relaxed:
                 warn("checksum should be unicode, not bytes",
                      PasslibHashWarning)
                 checksum = checksum.decode("ascii")
             else:
-                raise TypeError("checksum must be unicode string")
+                raise exc.ExpectedTypeError(checksum, "unicode", "checksum")
 
         # handle stub
         if checksum == self._stub_checksum:
@@ -960,14 +960,14 @@ class HasSalt(GenericHandler):
         # check type
         if self._salt_is_bytes:
             if not isinstance(salt, bytes):
-                raise TypeError("salt must be specified as bytes")
+                raise exc.ExpectedTypeError(salt, "bytes", "salt")
         else:
             if not isinstance(salt, unicode):
-                # XXX: should we disallow bytes here?
-                if isinstance(salt, bytes):
+                # NOTE: allowing bytes under py2 so salt can be native str.
+                if isinstance(salt, bytes) and (PY2 or self.relaxed):
                     salt = salt.decode("ascii")
                 else:
-                    raise TypeError("salt must be specified as unicode")
+                    raise exc.ExpectedTypeError(salt, "unicode", "salt")
 
             # check charset
             sc = self.salt_chars
@@ -1139,7 +1139,7 @@ class HasRounds(GenericHandler):
 
         # check type
         if not isinstance(rounds, int):
-            raise TypeError("rounds must be an integer")
+            raise exc.ExpectedTypeError(rounds, "integer", "rounds")
 
         # check bounds
         mn = self.min_rounds
diff --git a/passlib/utils/md4.py b/passlib/utils/md4.py
index cd3d012..4389a09 100644
--- a/passlib/utils/md4.py
+++ b/passlib/utils/md4.py
@@ -242,14 +242,14 @@ def _has_native_md4():
     try:
         h = hashlib.new("md4")
     except ValueError:
-        #not supported
+        # not supported - ssl probably missing (e.g. ironpython)
         return False
     result = h.hexdigest()
     if result == '31d6cfe0d16ae931b73c59d7e0c089c0':
         return True
     if PYPY and result == '':
-        #as of 1.5, pypy md4 just returns null!
-        #since this is expected, don't bother w/ warning.
+        # as of pypy 1.5-1.7, this returns empty string! 
+        # since this is expected, don't bother w/ warning.
         return False
     #anything else should alert user
     from passlib.exc import PasslibRuntimeWarning