pep 484 for types

strict types type_api.py, including TypeDecorator,
NativeForEmulated, etc.

Change-Id: Ib2eba26de0981324a83733954cb7044a29bbd7db

7 files changed, 194 insertions, 63 deletions

diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py
index 5b66c537a..061794bde 100644
--- a/lib/sqlalchemy/engine/base.py
+++ b/lib/sqlalchemy/engine/base.py
@@ -141,7 +141,7 @@ class Connection(ConnectionEventsTarget, inspection.Inspectable["Inspector"]):
         if connection is None:
             try:
                 self._dbapi_connection = engine.raw_connection()
-            except dialect.dbapi.Error as err:
+            except dialect.loaded_dbapi.Error as err:
                 Connection._handle_dbapi_exception_noconnection(
                     err, dialect, engine
                 )
@@ -1809,7 +1809,7 @@ class Connection(ConnectionEventsTarget, inspection.Inspectable["Inspector"]):
 
         if not self._is_disconnect:
             self._is_disconnect = (
-                isinstance(e, self.dialect.dbapi.Error)
+                isinstance(e, self.dialect.loaded_dbapi.Error)
                 and not self.closed
                 and self.dialect.is_disconnect(
                     e,
@@ -1825,7 +1825,7 @@ class Connection(ConnectionEventsTarget, inspection.Inspectable["Inspector"]):
                 statement,
                 parameters,
                 e,
-                self.dialect.dbapi.Error,
+                self.dialect.loaded_dbapi.Error,
                 hide_parameters=self.engine.hide_parameters,
                 dialect=self.dialect,
                 ismulti=context.executemany if context is not None else None,
@@ -1834,7 +1834,7 @@ class Connection(ConnectionEventsTarget, inspection.Inspectable["Inspector"]):
         try:
             # non-DBAPI error - if we already got a context,
             # or there's no string statement, don't wrap it
-            should_wrap = isinstance(e, self.dialect.dbapi.Error) or (
+            should_wrap = isinstance(e, self.dialect.loaded_dbapi.Error) or (
                 statement is not None
                 and context is None
                 and not is_exit_exception
@@ -1845,7 +1845,7 @@ class Connection(ConnectionEventsTarget, inspection.Inspectable["Inspector"]):
                     statement,
                     parameters,
                     cast(Exception, e),
-                    self.dialect.dbapi.Error,
+                    self.dialect.loaded_dbapi.Error,
                     hide_parameters=self.engine.hide_parameters,
                     connection_invalidated=self._is_disconnect,
                     dialect=self.dialect,
@@ -1943,17 +1943,17 @@ class Connection(ConnectionEventsTarget, inspection.Inspectable["Inspector"]):
         exc_info = sys.exc_info()
 
         is_disconnect = isinstance(
-            e, dialect.dbapi.Error
+            e, dialect.loaded_dbapi.Error
         ) and dialect.is_disconnect(e, None, None)
 
-        should_wrap = isinstance(e, dialect.dbapi.Error)
+        should_wrap = isinstance(e, dialect.loaded_dbapi.Error)
 
         if should_wrap:
             sqlalchemy_exception = exc.DBAPIError.instance(
                 None,
                 None,
                 cast(Exception, e),
-                dialect.dbapi.Error,
+                dialect.loaded_dbapi.Error,
                 hide_parameters=engine.hide_parameters,
                 connection_invalidated=is_disconnect,
             )
diff --git a/lib/sqlalchemy/engine/cursor.py b/lib/sqlalchemy/engine/cursor.py
index f776e5975..4f151e79c 100644
--- a/lib/sqlalchemy/engine/cursor.py
+++ b/lib/sqlalchemy/engine/cursor.py
@@ -23,6 +23,7 @@ from typing import List
 from typing import Optional
 from typing import Sequence
 from typing import Tuple
+from typing import TYPE_CHECKING
 from typing import Union
 
 from .result import MergedResult
@@ -57,7 +58,7 @@ if typing.TYPE_CHECKING:
     from .result import _KeyMapType
     from .result import _KeyType
     from .result import _ProcessorsType
-    from .result import _ProcessorType
+    from ..sql.type_api import _ResultProcessorType
 
 # metadata entry tuple indexes.
 # using raw tuple is faster than namedtuple.
@@ -77,7 +78,7 @@ MD_UNTRANSLATED: Literal[6] = 6  # raw name from cursor.description
 
 
 _CursorKeyMapRecType = Tuple[
-    int, int, List[Any], str, str, Optional["_ProcessorType"], str
+    int, int, List[Any], str, str, Optional["_ResultProcessorType"], str
 ]
 
 _CursorKeyMapType = Dict["_KeyType", _CursorKeyMapRecType]
@@ -164,6 +165,9 @@ class CursorResultMetaData(ResultMetaData):
         compiled_statement = context.compiled.statement
         invoked_statement = context.invoked_statement
 
+        if TYPE_CHECKING:
+            assert isinstance(invoked_statement, elements.ClauseElement)
+
         if compiled_statement is invoked_statement:
             return self
 
diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py
index ba34a0d42..4a833d2e5 100644
--- a/lib/sqlalchemy/engine/default.py
+++ b/lib/sqlalchemy/engine/default.py
@@ -57,6 +57,8 @@ from ..sql.compiler import SQLCompiler
 from ..sql.elements import quoted_name
 
 if typing.TYPE_CHECKING:
+    from types import ModuleType
+
     from .base import Connection
     from .base import Engine
     from .characteristics import ConnectionCharacteristic
@@ -67,8 +69,10 @@ if typing.TYPE_CHECKING:
     from .interfaces import _DBAPIMultiExecuteParams
     from .interfaces import _DBAPISingleExecuteParams
     from .interfaces import _ExecuteOptions
+    from .interfaces import _IsolationLevel
     from .interfaces import _MutableCoreSingleExecuteParams
-    from .result import _ProcessorType
+    from .interfaces import _ParamStyle
+    from .interfaces import DBAPIConnection
     from .row import Row
     from .url import URL
     from ..event import _ListenerFnType
@@ -76,12 +80,16 @@ if typing.TYPE_CHECKING:
     from ..pool import PoolProxiedConnection
     from ..sql import Executable
     from ..sql.compiler import Compiled
+    from ..sql.compiler import Linting
     from ..sql.compiler import ResultColumnsEntry
     from ..sql.compiler import TypeCompiler
     from ..sql.dml import DMLState
+    from ..sql.dml import UpdateBase
     from ..sql.elements import BindParameter
     from ..sql.schema import Column
     from ..sql.schema import ColumnDefault
+    from ..sql.type_api import _BindProcessorType
+    from ..sql.type_api import _ResultProcessorType
     from ..sql.type_api import TypeEngine
 
 # When we're handed literal SQL, ensure it's a SELECT query
@@ -102,10 +110,7 @@ class DefaultDialect(Dialect):
 
     statement_compiler = compiler.SQLCompiler
     ddl_compiler = compiler.DDLCompiler
-    if typing.TYPE_CHECKING:
-        type_compiler: TypeCompiler
-    else:
-        type_compiler = compiler.GenericTypeCompiler
+    type_compiler_cls = compiler.GenericTypeCompiler
 
     preparer = compiler.IdentifierPreparer
     supports_alter = True
@@ -253,20 +258,19 @@ class DefaultDialect(Dialect):
     )
     def __init__(
         self,
-        paramstyle=None,
-        isolation_level=None,
-        dbapi=None,
-        implicit_returning=None,
-        supports_native_boolean=None,
-        max_identifier_length=None,
-        label_length=None,
-        # int() is because the @deprecated_params decorator cannot accommodate
-        # the direct reference to the "NO_LINTING" object
-        compiler_linting=int(compiler.NO_LINTING),
-        server_side_cursors=False,
-        **kwargs,
+        paramstyle: Optional[_ParamStyle] = None,
+        isolation_level: Optional[_IsolationLevel] = None,
+        dbapi: Optional[ModuleType] = None,
+        implicit_returning: Optional[bool] = None,
+        supports_native_boolean: Optional[bool] = None,
+        max_identifier_length: Optional[int] = None,
+        label_length: Optional[int] = None,
+        # util.deprecated_params decorator cannot render the
+        # Linting.NO_LINTING constant
+        compiler_linting: Linting = int(compiler.NO_LINTING),  # type: ignore
+        server_side_cursors: bool = False,
+        **kwargs: Any,
     ):
-
         if server_side_cursors:
             if not self.supports_server_side_cursors:
                 raise exc.ArgumentError(
@@ -286,7 +290,9 @@ class DefaultDialect(Dialect):
 
         self.positional = False
         self._ischema = None
+
         self.dbapi = dbapi
+
         if paramstyle is not None:
             self.paramstyle = paramstyle
         elif self.dbapi is not None:
@@ -299,11 +305,17 @@ class DefaultDialect(Dialect):
         self.identifier_preparer = self.preparer(self)
         self._on_connect_isolation_level = isolation_level
 
-        tt_callable = cast(
-            Type[compiler.GenericTypeCompiler],
-            self.type_compiler,
-        )
-        self.type_compiler = tt_callable(self)
+        legacy_tt_callable = getattr(self, "type_compiler", None)
+        if legacy_tt_callable is not None:
+            tt_callable = cast(
+                Type[compiler.GenericTypeCompiler],
+                self.type_compiler,
+            )
+        else:
+            tt_callable = self.type_compiler_cls
+
+        self.type_compiler_instance = self.type_compiler = tt_callable(self)
+
         if supports_native_boolean is not None:
             self.supports_native_boolean = supports_native_boolean
 
@@ -316,6 +328,15 @@ class DefaultDialect(Dialect):
         self.compiler_linting = compiler_linting
 
     @util.memoized_property
+    def loaded_dbapi(self) -> ModuleType:
+        if self.dbapi is None:
+            raise exc.InvalidRequestError(
+                f"Dialect {self} does not have a Python DBAPI established "
+                "and cannot be used for actual database interaction"
+            )
+        return self.dbapi
+
+    @util.memoized_property
     def _bind_typing_render_casts(self):
         return self.bind_typing is interfaces.BindTyping.RENDER_CASTS
 
@@ -495,7 +516,7 @@ class DefaultDialect(Dialect):
 
     def connect(self, *cargs, **cparams):
         # inherits the docstring from interfaces.Dialect.connect
-        return self.dbapi.connect(*cargs, **cparams)
+        return self.loaded_dbapi.connect(*cargs, **cparams)
 
     def create_connect_args(self, url):
         # inherits the docstring from interfaces.Dialect.create_connect_args
@@ -584,7 +605,7 @@ class DefaultDialect(Dialect):
     def _dialect_specific_select_one(self):
         return str(expression.select(1).compile(dialect=self))
 
-    def do_ping(self, dbapi_connection):
+    def do_ping(self, dbapi_connection: DBAPIConnection) -> bool:
         cursor = None
         try:
             cursor = dbapi_connection.cursor()
@@ -592,7 +613,7 @@ class DefaultDialect(Dialect):
                 cursor.execute(self._dialect_specific_select_one)
             finally:
                 cursor.close()
-        except self.dbapi.Error as err:
+        except self.loaded_dbapi.Error as err:
             if self.is_disconnect(err, dbapi_connection, cursor):
                 return False
             else:
@@ -747,7 +768,7 @@ class StrCompileDialect(DefaultDialect):
 
     statement_compiler = compiler.StrSQLCompiler
     ddl_compiler = compiler.DDLCompiler
-    type_compiler = compiler.StrSQLTypeCompiler  # type: ignore
+    type_compiler_cls = compiler.StrSQLTypeCompiler
     preparer = compiler.IdentifierPreparer
 
     supports_statement_cache = True
@@ -906,6 +927,8 @@ class DefaultExecutionContext(ExecutionContext):
         self.is_text = compiled.isplaintext
 
         if self.isinsert or self.isupdate or self.isdelete:
+            if TYPE_CHECKING:
+                assert isinstance(compiled.statement, UpdateBase)
             self.is_crud = True
             self._is_explicit_returning = bool(compiled.statement._returning)
             self._is_implicit_returning = bool(
@@ -943,7 +966,7 @@ class DefaultExecutionContext(ExecutionContext):
         processors = compiled._bind_processors
 
         flattened_processors: Mapping[
-            str, _ProcessorType
+            str, _BindProcessorType[Any]
         ] = processors  # type: ignore[assignment]
 
         if compiled.literal_execute_params or compiled.post_compile_params:
@@ -1354,7 +1377,7 @@ class DefaultExecutionContext(ExecutionContext):
 
             type_ = bindparam.type
             impl_type = type_.dialect_impl(self.dialect)
-            dbapi_type = impl_type.get_dbapi_type(self.dialect.dbapi)
+            dbapi_type = impl_type.get_dbapi_type(self.dialect.loaded_dbapi)
             result_processor = impl_type.result_processor(
                 self.dialect, dbapi_type
             )
diff --git a/lib/sqlalchemy/engine/interfaces.py b/lib/sqlalchemy/engine/interfaces.py
index 3ca30d1bc..1b178641e 100644
--- a/lib/sqlalchemy/engine/interfaces.py
+++ b/lib/sqlalchemy/engine/interfaces.py
@@ -14,12 +14,14 @@ from types import ModuleType
 from typing import Any
 from typing import Awaitable
 from typing import Callable
+from typing import ClassVar
 from typing import Dict
 from typing import List
 from typing import Mapping
 from typing import MutableMapping
 from typing import Optional
 from typing import Sequence
+from typing import Set
 from typing import Tuple
 from typing import Type
 from typing import TYPE_CHECKING
@@ -60,6 +62,7 @@ if TYPE_CHECKING:
     from ..sql.schema import DefaultGenerator
     from ..sql.schema import Sequence as Sequence_SchemaItem
     from ..sql.sqltypes import Integer
+    from ..sql.type_api import _TypeMemoDict
     from ..sql.type_api import TypeEngine
 
 ConnectArgsType = Tuple[Tuple[str], MutableMapping[str, Any]]
@@ -166,7 +169,7 @@ class DBAPICursor(Protocol):
     def execute(
         self,
         operation: Any,
-        parameters: Optional[_DBAPISingleExecuteParams],
+        parameters: Optional[_DBAPISingleExecuteParams] = None,
     ) -> Any:
         ...
 
@@ -577,7 +580,7 @@ class Dialect(EventTarget):
     driver: str
     """identifying name for the dialect's DBAPI"""
 
-    dbapi: ModuleType
+    dbapi: Optional[ModuleType]
     """A reference to the DBAPI module object itself.
 
     SQLAlchemy dialects import DBAPI modules using the classmethod
@@ -600,6 +603,16 @@ class Dialect(EventTarget):
 
     """
 
+    @util.non_memoized_property
+    def loaded_dbapi(self) -> ModuleType:
+        """same as .dbapi, but is never None; will raise an error if no
+        DBAPI was set up.
+
+        .. versionadded:: 2.0
+
+        """
+        raise NotImplementedError()
+
     positional: bool
     """True if the paramstyle for this Dialect is positional."""
 
@@ -616,8 +629,28 @@ class Dialect(EventTarget):
     ddl_compiler: Type[DDLCompiler]
     """a :class:`.Compiled` class used to compile DDL statements"""
 
-    type_compiler: Union[Type[TypeCompiler], TypeCompiler]
-    """a :class:`.Compiled` class used to compile SQL type objects"""
+    type_compiler_cls: ClassVar[Type[TypeCompiler]]
+    """a :class:`.Compiled` class used to compile SQL type objects
+
+    .. versionadded:: 2.0
+
+    """
+
+    type_compiler_instance: TypeCompiler
+    """instance of a :class:`.Compiled` class used to compile SQL type
+    objects
+
+    .. versionadded:: 2.0
+
+    """
+
+    type_compiler: Any
+    """legacy; this is a TypeCompiler class at the class level, a
+    TypeCompiler instance at the instance level.
+
+    Refer to type_compiler_instance instead.
+
+    """
 
     preparer: Type[IdentifierPreparer]
     """a :class:`.IdentifierPreparer` class used to
@@ -683,9 +716,17 @@ class Dialect(EventTarget):
     """
 
     supports_default_values: bool
-    """Indicates if the construct ``INSERT INTO tablename DEFAULT
-      VALUES`` is supported
-    """
+    """dialect supports INSERT... DEFAULT VALUES syntax"""
+
+    supports_default_metavalue: bool
+    """dialect supports INSERT... VALUES (DEFAULT) syntax"""
+
+    supports_empty_insert: bool
+    """dialect supports INSERT () VALUES ()"""
+
+    supports_multivalues_insert: bool
+    """Target database supports INSERT...VALUES with multiple value
+    sets"""
 
     preexecute_autoincrement_sequences: bool
     """True if 'implicit' primary key functions must be executed separately
@@ -723,6 +764,12 @@ class Dialect(EventTarget):
       other backends.
     """
 
+    default_sequence_base: int
+    """the default value that will be rendered as the "START WITH" portion of
+    a CREATE SEQUENCE DDL statement.
+
+    """
+
     supports_native_enum: bool
     """Indicates if the dialect supports a native ENUM construct.
       This will prevent :class:`_types.Enum` from generating a CHECK
@@ -735,6 +782,10 @@ class Dialect(EventTarget):
       constraint when that type is used.
     """
 
+    supports_native_decimal: bool
+    """indicates if Decimal objects are handled and returned for precision
+    numeric types, or if floats are returned"""
+
     construct_arguments: Optional[
         List[Tuple[Type[ClauseElement], Mapping[str, Any]]]
     ] = None
@@ -842,6 +893,52 @@ class Dialect(EventTarget):
 
     """
 
+    label_length: Optional[int]
+    """optional user-defined max length for SQL labels"""
+
+    include_set_input_sizes: Optional[Set[Any]]
+    """set of DBAPI type objects that should be included in
+    automatic cursor.setinputsizes() calls.
+
+    This is only used if bind_typing is BindTyping.SET_INPUT_SIZES
+
+    """
+
+    exclude_set_input_sizes: Optional[Set[Any]]
+    """set of DBAPI type objects that should be excluded in
+    automatic cursor.setinputsizes() calls.
+
+    This is only used if bind_typing is BindTyping.SET_INPUT_SIZES
+
+    """
+
+    supports_simple_order_by_label: bool
+    """target database supports ORDER BY <labelname>, where <labelname>
+    refers to a label in the columns clause of the SELECT"""
+
+    div_is_floordiv: bool
+    """target database treats the / division operator as "floor division" """
+
+    tuple_in_values: bool
+    """target database supports tuple IN, i.e. (x, y) IN ((q, p), (r, z))"""
+
+    _bind_typing_render_casts: bool
+
+    supports_identity_columns: bool
+    """target database supports IDENTITY"""
+
+    cte_follows_insert: bool
+    """target database, when given a CTE with an INSERT statement, needs
+    the CTE to be below the INSERT"""
+
+    insert_executemany_returning: bool
+    """dialect / driver / database supports some means of providing RETURNING
+    support when dialect.do_executemany() is used.
+
+    """
+
+    _type_memos: MutableMapping[TypeEngine[Any], "_TypeMemoDict"]
+
     def _builtin_onconnect(self) -> Optional[_ListenerFnType]:
         raise NotImplementedError()
 
@@ -1495,7 +1592,7 @@ class Dialect(EventTarget):
     def is_disconnect(
         self,
         e: Exception,
-        connection: Optional[PoolProxiedConnection],
+        connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]],
         cursor: Optional[DBAPICursor],
     ) -> bool:
         """Return True if the given DB-API error indicates an invalid
diff --git a/lib/sqlalchemy/engine/processors.py b/lib/sqlalchemy/engine/processors.py
index 7a6a57c03..587926752 100644
--- a/lib/sqlalchemy/engine/processors.py
+++ b/lib/sqlalchemy/engine/processors.py
@@ -20,23 +20,29 @@ from ._py_processors import str_to_datetime_processor_factory  # noqa
 from ..util._has_cy import HAS_CYEXTENSION
 
 if typing.TYPE_CHECKING or not HAS_CYEXTENSION:
-    from ._py_processors import int_to_boolean  # noqa
-    from ._py_processors import str_to_date  # noqa
-    from ._py_processors import str_to_datetime  # noqa
-    from ._py_processors import str_to_time  # noqa
-    from ._py_processors import to_decimal_processor_factory  # noqa
-    from ._py_processors import to_float  # noqa
-    from ._py_processors import to_str  # noqa
+    from ._py_processors import int_to_boolean as int_to_boolean
+    from ._py_processors import str_to_date as str_to_date
+    from ._py_processors import str_to_datetime as str_to_datetime
+    from ._py_processors import str_to_time as str_to_time
+    from ._py_processors import (
+        to_decimal_processor_factory as to_decimal_processor_factory,
+    )
+    from ._py_processors import to_float as to_float
+    from ._py_processors import to_str as to_str
 else:
     from sqlalchemy.cyextension.processors import (
         DecimalResultProcessor,
     )  # noqa
-    from sqlalchemy.cyextension.processors import int_to_boolean  # noqa
-    from sqlalchemy.cyextension.processors import str_to_date  # noqa
-    from sqlalchemy.cyextension.processors import str_to_datetime  # noqa
-    from sqlalchemy.cyextension.processors import str_to_time  # noqa
-    from sqlalchemy.cyextension.processors import to_float  # noqa
-    from sqlalchemy.cyextension.processors import to_str  # noqa
+    from sqlalchemy.cyextension.processors import (
+        int_to_boolean as int_to_boolean,
+    )
+    from sqlalchemy.cyextension.processors import str_to_date as str_to_date
+    from sqlalchemy.cyextension.processors import (
+        str_to_datetime as str_to_datetime,
+    )
+    from sqlalchemy.cyextension.processors import str_to_time as str_to_time
+    from sqlalchemy.cyextension.processors import to_float as to_float
+    from sqlalchemy.cyextension.processors import to_str as to_str
 
     def to_decimal_processor_factory(target_class, scale):
         # Note that the scale argument is not taken into account for integer
diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py
index d428b8a9d..05b06e846 100644
--- a/lib/sqlalchemy/engine/result.py
+++ b/lib/sqlalchemy/engine/result.py
@@ -46,6 +46,7 @@ else:
 if typing.TYPE_CHECKING:
     from .row import RowMapping
     from ..sql.schema import Column
+    from ..sql.type_api import _ResultProcessorType
 
 _KeyType = Union[str, "Column[Any]"]
 _KeyIndexType = Union[str, "Column[Any]", int]
@@ -70,8 +71,7 @@ across all the result types
 
 """
 
-_ProcessorType = Callable[[Any], Any]
-_ProcessorsType = Sequence[Optional[_ProcessorType]]
+_ProcessorsType = Sequence[Optional["_ResultProcessorType[Any]"]]
 _TupleGetterType = Callable[[Sequence[Any]], Tuple[Any, ...]]
 _UniqueFilterType = Callable[[Any], Any]
 _UniqueFilterStateType = Tuple[Set[Any], Optional[_UniqueFilterType]]
diff --git a/lib/sqlalchemy/engine/row.py b/lib/sqlalchemy/engine/row.py
index ff63199d4..4ba39b55d 100644
--- a/lib/sqlalchemy/engine/row.py
+++ b/lib/sqlalchemy/engine/row.py
@@ -41,6 +41,7 @@ else:
 if typing.TYPE_CHECKING:
     from .result import _KeyType
     from .result import RMKeyView
+    from ..sql.type_api import _ResultProcessorType
 
 
 class Row(BaseRow, typing.Sequence[Any]):
@@ -105,7 +106,7 @@ class Row(BaseRow, typing.Sequence[Any]):
         )
 
     def _filter_on_values(
-        self, filters: Optional[Sequence[Optional[Callable[[Any], Any]]]]
+        self, filters: Optional[Sequence[Optional[_ResultProcessorType[Any]]]]
     ) -> Row:
         return Row(
             self._parent,