Reduce import time overhead

* Fix subclass traversals to not run classes multiple times

* switch compiler visitor to use an attrgetter, to avoid
  an eval() at startup time

* don't pre-generate traversal functions, there's lots of these
  which are expensive to generate at once and most applications
  won't use them all; have it generate them on first use instead

* Some ideas about removing asyncio imports, they don't seem to
  be too signficant, apply some more simplicity to the overall
  "greenlet fallback" situation

Fixes: #5681
Change-Id: Ib564ddaddb374787ce3e11ff48026e99ed570933

5 files changed, 157 insertions, 127 deletions

diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py
index 00066cbed..7c5257b87 100644
--- a/lib/sqlalchemy/util/__init__.py
+++ b/lib/sqlalchemy/util/__init__.py
@@ -157,6 +157,7 @@ from .langhelpers import set_creation_order  # noqa
 from .langhelpers import string_or_unprintable  # noqa
 from .langhelpers import symbol  # noqa
 from .langhelpers import unbound_method_to_callable  # noqa
+from .langhelpers import walk_subclasses  # noqa
 from .langhelpers import warn  # noqa
 from .langhelpers import warn_exception  # noqa
 from .langhelpers import warn_limited  # noqa
diff --git a/lib/sqlalchemy/util/_concurrency_py3k.py b/lib/sqlalchemy/util/_concurrency_py3k.py
index ee3abe5fa..5d11bf92c 100644
--- a/lib/sqlalchemy/util/_concurrency_py3k.py
+++ b/lib/sqlalchemy/util/_concurrency_py3k.py
@@ -4,110 +4,103 @@ from typing import Any
 from typing import Callable
 from typing import Coroutine
 
+import greenlet
+
 from .. import exc
 
-try:
-    import greenlet
 
-    # implementation based on snaury gist at
-    # https://gist.github.com/snaury/202bf4f22c41ca34e56297bae5f33fef
-    # Issue for context: https://github.com/python-greenlet/greenlet/issues/173
+# implementation based on snaury gist at
+# https://gist.github.com/snaury/202bf4f22c41ca34e56297bae5f33fef
+# Issue for context: https://github.com/python-greenlet/greenlet/issues/173
+
+
+class _AsyncIoGreenlet(greenlet.greenlet):
+    def __init__(self, fn, driver):
+        greenlet.greenlet.__init__(self, fn, driver)
+        self.driver = driver
+
+
+def await_only(awaitable: Coroutine) -> Any:
+    """Awaits an async function in a sync method.
+
+    The sync method must be insice a :func:`greenlet_spawn` context.
+    :func:`await_` calls cannot be nested.
 
-    class _AsyncIoGreenlet(greenlet.greenlet):
-        def __init__(self, fn, driver):
-            greenlet.greenlet.__init__(self, fn, driver)
-            self.driver = driver
+    :param awaitable: The coroutine to call.
 
-    def await_only(awaitable: Coroutine) -> Any:
-        """Awaits an async function in a sync method.
+    """
+    # this is called in the context greenlet while running fn
+    current = greenlet.getcurrent()
+    if not isinstance(current, _AsyncIoGreenlet):
+        raise exc.InvalidRequestError(
+            "greenlet_spawn has not been called; can't call await_() here."
+        )
 
-        The sync method must be insice a :func:`greenlet_spawn` context.
-        :func:`await_` calls cannot be nested.
+    # returns the control to the driver greenlet passing it
+    # a coroutine to run. Once the awaitable is done, the driver greenlet
+    # switches back to this greenlet with the result of awaitable that is
+    # then returned to the caller (or raised as error)
+    return current.driver.switch(awaitable)
 
-        :param awaitable: The coroutine to call.
 
-        """
-        # this is called in the context greenlet while running fn
-        current = greenlet.getcurrent()
-        if not isinstance(current, _AsyncIoGreenlet):
+def await_fallback(awaitable: Coroutine) -> Any:
+    """Awaits an async function in a sync method.
+
+    The sync method must be insice a :func:`greenlet_spawn` context.
+    :func:`await_` calls cannot be nested.
+
+    :param awaitable: The coroutine to call.
+
+    """
+
+    # this is called in the context greenlet while running fn
+    current = greenlet.getcurrent()
+    if not isinstance(current, _AsyncIoGreenlet):
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
             raise exc.InvalidRequestError(
-                "greenlet_spawn has not been called; can't call await_() here."
+                "greenlet_spawn has not been called and asyncio event "
+                "loop is already running; can't call await_() here."
             )
-
-        # returns the control to the driver greenlet passing it
-        # a coroutine to run. Once the awaitable is done, the driver greenlet
-        # switches back to this greenlet with the result of awaitable that is
-        # then returned to the caller (or raised as error)
-        return current.driver.switch(awaitable)
-
-    def await_fallback(awaitable: Coroutine) -> Any:
-        """Awaits an async function in a sync method.
-
-        The sync method must be insice a :func:`greenlet_spawn` context.
-        :func:`await_` calls cannot be nested.
-
-        :param awaitable: The coroutine to call.
-
-        """
-        # this is called in the context greenlet while running fn
-        current = greenlet.getcurrent()
-        if not isinstance(current, _AsyncIoGreenlet):
-            loop = asyncio.get_event_loop()
-            if loop.is_running():
-                raise exc.InvalidRequestError(
-                    "greenlet_spawn has not been called and asyncio event "
-                    "loop is already running; can't call await_() here."
-                )
-            return loop.run_until_complete(awaitable)
-
-        return current.driver.switch(awaitable)
-
-    async def greenlet_spawn(fn: Callable, *args, **kwargs) -> Any:
-        """Runs a sync function ``fn`` in a new greenlet.
-
-        The sync function can then use :func:`await_` to wait for async
-        functions.
-
-        :param fn: The sync callable to call.
-        :param \\*args: Positional arguments to pass to the ``fn`` callable.
-        :param \\*\\*kwargs: Keyword arguments to pass to the ``fn`` callable.
-        """
-        context = _AsyncIoGreenlet(fn, greenlet.getcurrent())
-        # runs the function synchronously in gl greenlet. If the execution
-        # is interrupted by await_, context is not dead and result is a
-        # coroutine to wait. If the context is dead the function has
-        # returned, and its result can be returned.
-        try:
-            result = context.switch(*args, **kwargs)
-            while not context.dead:
-                try:
-                    # wait for a coroutine from await_ and then return its
-                    # result back to it.
-                    value = await result
-                except Exception:
-                    # this allows an exception to be raised within
-                    # the moderated greenlet so that it can continue
-                    # its expected flow.
-                    result = context.throw(*sys.exc_info())
-                else:
-                    result = context.switch(value)
-        finally:
-            # clean up to avoid cycle resolution by gc
-            del context.driver
-        return result
-
-
-except ImportError:  # pragma: no cover
-    greenlet = None
-
-    def await_fallback(awaitable):
-        return asyncio.get_event_loop().run_until_complete(awaitable)
-
-    def await_only(awaitable):
-        raise ValueError("Greenlet is required to use this function")
-
-    async def greenlet_spawn(fn, *args, **kw):
-        raise ValueError("Greenlet is required to use this function")
+        return loop.run_until_complete(awaitable)
+
+    return current.driver.switch(awaitable)
+
+
+async def greenlet_spawn(fn: Callable, *args, **kwargs) -> Any:
+    """Runs a sync function ``fn`` in a new greenlet.
+
+    The sync function can then use :func:`await_` to wait for async
+    functions.
+
+    :param fn: The sync callable to call.
+    :param \\*args: Positional arguments to pass to the ``fn`` callable.
+    :param \\*\\*kwargs: Keyword arguments to pass to the ``fn`` callable.
+    """
+
+    context = _AsyncIoGreenlet(fn, greenlet.getcurrent())
+    # runs the function synchronously in gl greenlet. If the execution
+    # is interrupted by await_, context is not dead and result is a
+    # coroutine to wait. If the context is dead the function has
+    # returned, and its result can be returned.
+    try:
+        result = context.switch(*args, **kwargs)
+        while not context.dead:
+            try:
+                # wait for a coroutine from await_ and then return its
+                # result back to it.
+                value = await result
+            except Exception:
+                # this allows an exception to be raised within
+                # the moderated greenlet so that it can continue
+                # its expected flow.
+                result = context.throw(*sys.exc_info())
+            else:
+                result = context.switch(value)
+    finally:
+        # clean up to avoid cycle resolution by gc
+        del context.driver
+    return result
 
 
 class AsyncAdaptedLock:
diff --git a/lib/sqlalchemy/util/_preloaded.py b/lib/sqlalchemy/util/_preloaded.py
index 1a833a963..e267c008c 100644
--- a/lib/sqlalchemy/util/_preloaded.py
+++ b/lib/sqlalchemy/util/_preloaded.py
@@ -35,8 +35,9 @@ class _ModuleRegistry:
     name. Example: ``sqlalchemy.sql.util`` becomes ``preloaded.sql_util``.
     """
 
-    def __init__(self, prefix="sqlalchemy"):
+    def __init__(self, prefix="sqlalchemy."):
         self.module_registry = set()
+        self.prefix = prefix
 
     def preload_module(self, *deps):
         """Adds the specified modules to the list to load.
@@ -52,8 +53,13 @@ class _ModuleRegistry:
         specified path.
         """
         for module in self.module_registry:
-            key = module.split("sqlalchemy.")[-1].replace(".", "_")
-            if module.startswith(path) and key not in self.__dict__:
+            if self.prefix:
+                key = module.split(self.prefix)[-1].replace(".", "_")
+            else:
+                key = module
+            if (
+                not path or module.startswith(path)
+            ) and key not in self.__dict__:
                 compat.import_(module, globals(), locals())
                 self.__dict__[key] = sys.modules[module]
 
diff --git a/lib/sqlalchemy/util/concurrency.py b/lib/sqlalchemy/util/concurrency.py
index e0883aa68..f78c0971c 100644
--- a/lib/sqlalchemy/util/concurrency.py
+++ b/lib/sqlalchemy/util/concurrency.py
@@ -1,25 +1,40 @@
 from . import compat
 
+have_greenlet = False
 
 if compat.py3k:
-    import asyncio
-    from ._concurrency_py3k import await_only
-    from ._concurrency_py3k import await_fallback
-    from ._concurrency_py3k import greenlet
-    from ._concurrency_py3k import greenlet_spawn
-    from ._concurrency_py3k import AsyncAdaptedLock
-else:
-    asyncio = None
-    greenlet = None
-
-    def await_only(thing):
+    try:
+        import greenlet  # noqa F401
+    except ImportError:
+        pass
+    else:
+        have_greenlet = True
+        from ._concurrency_py3k import await_only
+        from ._concurrency_py3k import await_fallback
+        from ._concurrency_py3k import greenlet_spawn
+        from ._concurrency_py3k import AsyncAdaptedLock
+        from ._concurrency_py3k import asyncio  # noqa F401
+
+if not have_greenlet:
+
+    asyncio = None  # noqa F811
+
+    def _not_implemented():
+        if not compat.py3k:
+            raise ValueError("Cannot use this function in py2.")
+        else:
+            raise ValueError(
+                "the greenlet library is required to use this function."
+            )
+
+    def await_only(thing):  # noqa F811
         return thing
 
-    def await_fallback(thing):
+    def await_fallback(thing):  # noqa F81
         return thing
 
-    def greenlet_spawn(fn, *args, **kw):
-        raise ValueError("Cannot use this function in py2.")
+    def greenlet_spawn(fn, *args, **kw):  # noqa F81
+        _not_implemented()
 
-    def AsyncAdaptedLock(*args, **kw):
-        raise ValueError("Cannot use this function in py2.")
+    def AsyncAdaptedLock(*args, **kw):  # noqa F81
+        _not_implemented()
diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py
index 4289db812..ce3a28499 100644
--- a/lib/sqlalchemy/util/langhelpers.py
+++ b/lib/sqlalchemy/util/langhelpers.py
@@ -10,6 +10,7 @@ modules, classes, hierarchies, attributes, functions, and methods.
 
 """
 
+import collections
 from functools import update_wrapper
 import hashlib
 import inspect
@@ -83,6 +84,20 @@ class safe_reraise(object):
             compat.raise_(value, with_traceback=traceback)
 
 
+def walk_subclasses(cls):
+    seen = set()
+
+    stack = [cls]
+    while stack:
+        cls = stack.pop()
+        if cls in seen:
+            continue
+        else:
+            seen.add(cls)
+        stack.extend(cls.__subclasses__())
+        yield cls
+
+
 def string_or_unprintable(element):
     if isinstance(element, compat.string_types):
         return element
@@ -1782,15 +1797,22 @@ def inject_docstring_text(doctext, injecttext, pos):
     return "\n".join(lines)
 
 
+_param_reg = re.compile(r"(\s+):param (.+?):")
+
+
 def inject_param_text(doctext, inject_params):
-    doclines = doctext.splitlines()
+    doclines = collections.deque(doctext.splitlines())
     lines = []
 
+    # TODO: this is not working for params like ":param case_sensitive=True:"
+
     to_inject = None
     while doclines:
-        line = doclines.pop(0)
+        line = doclines.popleft()
+
+        m = _param_reg.match(line)
+
         if to_inject is None:
-            m = re.match(r"(\s+):param (.+?):", line)
             if m:
                 param = m.group(2).lstrip("*")
                 if param in inject_params:
@@ -1805,23 +1827,16 @@ def inject_param_text(doctext, inject_params):
                             indent = " " * len(m2.group(1))
 
                     to_inject = indent + inject_params[param]
-        elif line.lstrip().startswith(":param "):
-            lines.append("\n")
-            lines.append(to_inject)
-            lines.append("\n")
+        elif m:
+            lines.extend(["\n", to_inject, "\n"])
             to_inject = None
         elif not line.rstrip():
-            lines.append(line)
-            lines.append(to_inject)
-            lines.append("\n")
+            lines.extend([line, to_inject, "\n"])
             to_inject = None
         elif line.endswith("::"):
             # TODO: this still wont cover if the code example itself has blank
             # lines in it, need to detect those via indentation.
-            lines.append(line)
-            lines.append(
-                doclines.pop(0)
-            )  # the blank line following a code example
+            lines.extend([line, doclines.popleft()])
             continue
         lines.append(line)