diff options
| author | Gaëtan de Menten <gdementen@gmail.com> | 2010-02-13 22:53:39 +0000 |
|---|---|---|
| committer | Gaëtan de Menten <gdementen@gmail.com> | 2010-02-13 22:53:39 +0000 |
| commit | 165609a190665f5453417c9c935a834714c7f5a5 (patch) | |
| tree | 90d3d0da3f233cf6fc211f367eea0dba661b098e /lib/sqlalchemy/dialects | |
| parent | f2974ef3993e02646a2dfade5feb74afb78f370f (diff) | |
| download | sqlalchemy-165609a190665f5453417c9c935a834714c7f5a5.tar.gz | |
- Added an optional C extension to speed up the sql layer by
reimplementing the highest impact functions.
The actual speedups will depend heavily on your DBAPI and
the mix of datatypes used in your tables, and can vary from
a 50% improvement to more than 200%. It also provides a modest
(~20%) indirect improvement to ORM speed for large queries.
Note that it is *not* built/installed by default.
See README for installation instructions.
- The most common result processors conversion function were
moved to the new "processors" module. Dialect authors are
encouraged to use those functions whenever they correspond
to their needs instead of implementing custom ones.
Diffstat (limited to 'lib/sqlalchemy/dialects')
| -rw-r--r-- | lib/sqlalchemy/dialects/access/base.py | 45 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/informix/base.py | 2 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/maxdb/base.py | 122 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mssql/base.py | 17 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/base.py | 14 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/mysqldb.py | 8 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/oursql.py | 8 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/oracle/zxjdbc.py | 3 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/postgresql/pg8000.py | 25 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/postgresql/psycopg2.py | 19 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/postgresql/pypostgresql.py | 8 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/sqlite/base.py | 44 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/sybase/base.py | 19 |
13 files changed, 133 insertions, 201 deletions
diff --git a/lib/sqlalchemy/dialects/access/base.py b/lib/sqlalchemy/dialects/access/base.py index a46ad247a..c10e77011 100644 --- a/lib/sqlalchemy/dialects/access/base.py +++ b/lib/sqlalchemy/dialects/access/base.py @@ -17,23 +17,17 @@ This dialect is *not* tested on SQLAlchemy 0.6. from sqlalchemy import sql, schema, types, exc, pool from sqlalchemy.sql import compiler, expression from sqlalchemy.engine import default, base - +from sqlalchemy import processors class AcNumeric(types.Numeric): - def result_processor(self, dialect, coltype): - return None + def get_col_spec(self): + return "NUMERIC" def bind_processor(self, dialect): - def process(value): - if value is None: - # Not sure that this exception is needed - return value - else: - return str(value) - return process + return processors.to_str - def get_col_spec(self): - return "NUMERIC" + def result_processor(self, dialect, coltype): + return None class AcFloat(types.Float): def get_col_spec(self): @@ -41,11 +35,7 @@ class AcFloat(types.Float): def bind_processor(self, dialect): """By converting to string, we can use Decimal types round-trip.""" - def process(value): - if not value is None: - return str(value) - return None - return process + return processors.to_str class AcInteger(types.Integer): def get_col_spec(self): @@ -103,25 +93,6 @@ class AcBoolean(types.Boolean): def get_col_spec(self): return "YESNO" - def result_processor(self, dialect, coltype): - def process(value): - if value is None: - return None - return value and True or False - return process - - def bind_processor(self, dialect): - def process(value): - if value is True: - return 1 - elif value is False: - return 0 - elif value is None: - return None - else: - return value and True or False - return process - class AcTimeStamp(types.TIMESTAMP): def get_col_spec(self): return "TIMESTAMP" @@ -443,4 +414,4 @@ dialect.poolclass = pool.SingletonThreadPool dialect.statement_compiler = AccessCompiler dialect.ddlcompiler = AccessDDLCompiler dialect.preparer = AccessIdentifierPreparer -dialect.execution_ctx_cls = AccessExecutionContext
\ No newline at end of file +dialect.execution_ctx_cls = AccessExecutionContext diff --git a/lib/sqlalchemy/dialects/informix/base.py b/lib/sqlalchemy/dialects/informix/base.py index 2802d493a..54aae6eb3 100644 --- a/lib/sqlalchemy/dialects/informix/base.py +++ b/lib/sqlalchemy/dialects/informix/base.py @@ -302,4 +302,4 @@ class InformixDialect(default.DefaultDialect): @reflection.cache def get_indexes(self, connection, table_name, schema, **kw): # TODO - return []
\ No newline at end of file + return [] diff --git a/lib/sqlalchemy/dialects/maxdb/base.py b/lib/sqlalchemy/dialects/maxdb/base.py index 2e0b9518b..f409f3213 100644 --- a/lib/sqlalchemy/dialects/maxdb/base.py +++ b/lib/sqlalchemy/dialects/maxdb/base.py @@ -60,7 +60,7 @@ this. """ import datetime, itertools, re -from sqlalchemy import exc, schema, sql, util +from sqlalchemy import exc, schema, sql, util, processors from sqlalchemy.sql import operators as sql_operators, expression as sql_expr from sqlalchemy.sql import compiler, visitors from sqlalchemy.engine import base as engine_base, default @@ -86,6 +86,12 @@ class _StringType(sqltypes.String): return process def result_processor(self, dialect, coltype): + #XXX: this code is probably very slow and one should try (if at all + # possible) to determine the correct code path on a per-connection + # basis (ie, here in result_processor, instead of inside the processor + # function itself) and probably also use a few generic + # processors, or possibly per query (though there is no mechanism + # for that yet). def process(value): while True: if value is None: @@ -152,6 +158,7 @@ class MaxNumeric(sqltypes.Numeric): def bind_processor(self, dialect): return None + class MaxTimestamp(sqltypes.DateTime): def bind_processor(self, dialect): def process(value): @@ -172,25 +179,30 @@ class MaxTimestamp(sqltypes.DateTime): return process def result_processor(self, dialect, coltype): - def process(value): - if value is None: - return None - elif dialect.datetimeformat == 'internal': - return datetime.datetime( - *[int(v) - for v in (value[0:4], value[4:6], value[6:8], - value[8:10], value[10:12], value[12:14], - value[14:])]) - elif dialect.datetimeformat == 'iso': - return datetime.datetime( - *[int(v) - for v in (value[0:4], value[5:7], value[8:10], - value[11:13], value[14:16], value[17:19], - value[20:])]) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % ( - dialect.datetimeformat,)) + if dialect.datetimeformat == 'internal': + def process(value): + if value is None: + return None + else: + return datetime.datetime( + *[int(v) + for v in (value[0:4], value[4:6], value[6:8], + value[8:10], value[10:12], value[12:14], + value[14:])]) + elif dialect.datetimeformat == 'iso': + def process(value): + if value is None: + return None + else: + return datetime.datetime( + *[int(v) + for v in (value[0:4], value[5:7], value[8:10], + value[11:13], value[14:16], value[17:19], + value[20:])]) + else: + raise exc.InvalidRequestError( + "datetimeformat '%s' is not supported." % + dialect.datetimeformat) return process @@ -212,19 +224,24 @@ class MaxDate(sqltypes.Date): return process def result_processor(self, dialect, coltype): - def process(value): - if value is None: - return None - elif dialect.datetimeformat == 'internal': - return datetime.date( - *[int(v) for v in (value[0:4], value[4:6], value[6:8])]) - elif dialect.datetimeformat == 'iso': - return datetime.date( - *[int(v) for v in (value[0:4], value[5:7], value[8:10])]) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % ( - dialect.datetimeformat,)) + if dialect.datetimeformat == 'internal': + def process(value): + if value is None: + return None + else: + return datetime.date(int(value[0:4]), int(value[4:6]), + int(value[6:8])) + elif dialect.datetimeformat == 'iso': + def process(value): + if value is None: + return None + else: + return datetime.date(int(value[0:4]), int(value[5:7]), + int(value[8:10])) + else: + raise exc.InvalidRequestError( + "datetimeformat '%s' is not supported." % + dialect.datetimeformat) return process @@ -246,31 +263,30 @@ class MaxTime(sqltypes.Time): return process def result_processor(self, dialect, coltype): - def process(value): - if value is None: - return None - elif dialect.datetimeformat == 'internal': - t = datetime.time( - *[int(v) for v in (value[0:4], value[4:6], value[6:8])]) - return t - elif dialect.datetimeformat == 'iso': - return datetime.time( - *[int(v) for v in (value[0:4], value[5:7], value[8:10])]) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % ( - dialect.datetimeformat,)) + if dialect.datetimeformat == 'internal': + def process(value): + if value is None: + return None + else: + return datetime.time(int(value[0:4]), int(value[4:6]), + int(value[6:8])) + elif dialect.datetimeformat == 'iso': + def process(value): + if value is None: + return None + else: + return datetime.time(int(value[0:4]), int(value[5:7]), + int(value[8:10])) + else: + raise exc.InvalidRequestError( + "datetimeformat '%s' is not supported." % + dialect.datetimeformat) return process class MaxBlob(sqltypes.LargeBinary): def bind_processor(self, dialect): - def process(value): - if value is None: - return None - else: - return str(value) - return process + return processors.to_str def result_processor(self, dialect, coltype): def process(value): diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index 4e58d64b3..3f4e0b9f3 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -233,11 +233,10 @@ from sqlalchemy.sql import select, compiler, expression, \ functions as sql_functions, util as sql_util from sqlalchemy.engine import default, base, reflection from sqlalchemy import types as sqltypes -from decimal import Decimal as _python_Decimal +from sqlalchemy import processors from sqlalchemy.types import INTEGER, BIGINT, SMALLINT, DECIMAL, NUMERIC, \ FLOAT, TIMESTAMP, DATETIME, DATE, BINARY,\ VARBINARY, BLOB - from sqlalchemy.dialects.mssql import information_schema as ischema @@ -280,22 +279,12 @@ RESERVED_WORDS = set( class _MSNumeric(sqltypes.Numeric): def result_processor(self, dialect, coltype): if self.asdecimal: - def process(value): - if value is not None: - return _python_Decimal(str(value)) - else: - return value - return process + return processors.to_decimal_processor_factory(decimal.Decimal) else: #XXX: if the DBAPI returns a float (this is likely, given the # processor when asdecimal is True), this should be a None # processor instead. - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float def bind_processor(self, dialect): def process(value): diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index eb348f1a1..82a4af941 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -351,7 +351,8 @@ class DECIMAL(_NumericType, sqltypes.DECIMAL): numeric. """ - super(DECIMAL, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + super(DECIMAL, self).__init__(precision=precision, scale=scale, + asdecimal=asdecimal, **kw) class DOUBLE(_FloatType): @@ -375,7 +376,8 @@ class DOUBLE(_FloatType): numeric. """ - super(DOUBLE, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + super(DOUBLE, self).__init__(precision=precision, scale=scale, + asdecimal=asdecimal, **kw) class REAL(_FloatType): """MySQL REAL type.""" @@ -398,7 +400,8 @@ class REAL(_FloatType): numeric. """ - super(REAL, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + super(REAL, self).__init__(precision=precision, scale=scale, + asdecimal=asdecimal, **kw) class FLOAT(_FloatType, sqltypes.FLOAT): """MySQL FLOAT type.""" @@ -421,7 +424,8 @@ class FLOAT(_FloatType, sqltypes.FLOAT): numeric. """ - super(FLOAT, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + super(FLOAT, self).__init__(precision=precision, scale=scale, + asdecimal=asdecimal, **kw) def bind_processor(self, dialect): return None @@ -2459,6 +2463,7 @@ class _DecodingRowProxy(object): def __init__(self, rowproxy, charset): self.rowproxy = rowproxy self.charset = charset + def __getitem__(self, index): item = self.rowproxy[index] if isinstance(item, _array): @@ -2467,6 +2472,7 @@ class _DecodingRowProxy(object): return item.decode(self.charset) else: return item + def __getattr__(self, attr): item = getattr(self.rowproxy, attr) if isinstance(item, _array): diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index c07ed8713..8cfd5930f 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -28,6 +28,7 @@ from sqlalchemy.dialects.mysql.base import (DECIMAL, MySQLDialect, MySQLExecutio from sqlalchemy.engine import base as engine_base, default from sqlalchemy.sql import operators as sql_operators from sqlalchemy import exc, log, schema, sql, types as sqltypes, util +from sqlalchemy import processors class MySQL_mysqldbExecutionContext(MySQLExecutionContext): @@ -51,12 +52,7 @@ class _DecimalType(_NumericType): def result_processor(self, dialect, coltype): if self.asdecimal: return None - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float class _MySQLdbNumeric(_DecimalType, NUMERIC): diff --git a/lib/sqlalchemy/dialects/mysql/oursql.py b/lib/sqlalchemy/dialects/mysql/oursql.py index a03aa988e..1fca6850a 100644 --- a/lib/sqlalchemy/dialects/mysql/oursql.py +++ b/lib/sqlalchemy/dialects/mysql/oursql.py @@ -29,18 +29,14 @@ from sqlalchemy.dialects.mysql.base import (BIT, MySQLDialect, MySQLExecutionCon from sqlalchemy.engine import base as engine_base, default from sqlalchemy.sql import operators as sql_operators from sqlalchemy import exc, log, schema, sql, types as sqltypes, util +from sqlalchemy import processors class _oursqlNumeric(NUMERIC): def result_processor(self, dialect, coltype): if self.asdecimal: return None - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float class _oursqlBIT(BIT): diff --git a/lib/sqlalchemy/dialects/oracle/zxjdbc.py b/lib/sqlalchemy/dialects/oracle/zxjdbc.py index 22a1f443c..fba16288a 100644 --- a/lib/sqlalchemy/dialects/oracle/zxjdbc.py +++ b/lib/sqlalchemy/dialects/oracle/zxjdbc.py @@ -32,6 +32,9 @@ class _ZxJDBCDate(sqltypes.Date): class _ZxJDBCNumeric(sqltypes.Numeric): def result_processor(self, dialect, coltype): + #XXX: does the dialect return Decimal or not??? + # if it does (in all cases), we could use a None processor as well as + # the to_float generic processor if self.asdecimal: def process(value): if isinstance(value, decimal.Decimal): diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index e90bebb6b..079b05530 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -19,31 +19,23 @@ Interval Passing data from/to the Interval type is not supported as of yet. """ -from sqlalchemy.engine import default import decimal + +from sqlalchemy.engine import default from sqlalchemy import util, exc +from sqlalchemy import processors from sqlalchemy import types as sqltypes from sqlalchemy.dialects.postgresql.base import PGDialect, \ PGCompiler, PGIdentifierPreparer, PGExecutionContext class _PGNumeric(sqltypes.Numeric): def bind_processor(self, dialect): - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float def result_processor(self, dialect, coltype): if self.asdecimal: if coltype in (700, 701): - def process(value): - if value is not None: - return decimal.Decimal(str(value)) - else: - return value - return process + return processors.to_decimal_processor_factory(decimal.Decimal) elif coltype == 1700: # pg8000 returns Decimal natively for 1700 return None @@ -54,12 +46,7 @@ class _PGNumeric(sqltypes.Numeric): # pg8000 returns float natively for 701 return None elif coltype == 1700: - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float else: raise exc.InvalidRequestError("Unknown PG numeric type: %d" % coltype) diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index bb6562dea..712124288 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -46,8 +46,11 @@ The following per-statement execution options are respected: """ -import decimal, random, re +import random, re +import decimal + from sqlalchemy import util +from sqlalchemy import processors from sqlalchemy.engine import base, default from sqlalchemy.sql import expression from sqlalchemy.sql import operators as sql_operators @@ -63,12 +66,7 @@ class _PGNumeric(sqltypes.Numeric): def result_processor(self, dialect, coltype): if self.asdecimal: if coltype in (700, 701): - def process(value): - if value is not None: - return decimal.Decimal(str(value)) - else: - return value - return process + return processors.to_decimal_processor_factory(decimal.Decimal) elif coltype == 1700: # pg8000 returns Decimal natively for 1700 return None @@ -79,12 +77,7 @@ class _PGNumeric(sqltypes.Numeric): # pg8000 returns float natively for 701 return None elif coltype == 1700: - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float else: raise exc.InvalidRequestError("Unknown PG numeric type: %d" % coltype) diff --git a/lib/sqlalchemy/dialects/postgresql/pypostgresql.py b/lib/sqlalchemy/dialects/postgresql/pypostgresql.py index 77ed44512..88f1acde7 100644 --- a/lib/sqlalchemy/dialects/postgresql/pypostgresql.py +++ b/lib/sqlalchemy/dialects/postgresql/pypostgresql.py @@ -12,6 +12,7 @@ import decimal from sqlalchemy import util from sqlalchemy import types as sqltypes from sqlalchemy.dialects.postgresql.base import PGDialect, PGExecutionContext +from sqlalchemy import processors class PGNumeric(sqltypes.Numeric): def bind_processor(self, dialect): @@ -21,12 +22,7 @@ class PGNumeric(sqltypes.Numeric): if self.asdecimal: return None else: - def process(value): - if value is not None: - return float(value) - else: - return value - return process + return processors.to_float class PostgreSQL_pypostgresqlExecutionContext(PGExecutionContext): pass diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index 696f65a6c..e987439c5 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -54,6 +54,7 @@ from sqlalchemy import types as sqltypes from sqlalchemy import util from sqlalchemy.sql import compiler, functions as sql_functions from sqlalchemy.util import NoneType +from sqlalchemy import processors from sqlalchemy.types import BLOB, BOOLEAN, CHAR, DATE, DATETIME, DECIMAL,\ FLOAT, INTEGER, NUMERIC, SMALLINT, TEXT, TIME,\ @@ -62,13 +63,10 @@ from sqlalchemy.types import BLOB, BOOLEAN, CHAR, DATE, DATETIME, DECIMAL,\ class _NumericMixin(object): def bind_processor(self, dialect): - type_ = self.asdecimal and str or float - def process(value): - if value is not None: - return type_(value) - else: - return value - return process + if self.asdecimal: + return processors.to_str + else: + return processors.to_float class _SLNumeric(_NumericMixin, sqltypes.Numeric): pass @@ -86,19 +84,7 @@ class _DateTimeMixin(object): if storage_format is not None: self._storage_format = storage_format - def _result_processor(self, fn): - rmatch = self._reg.match - # Even on python2.6 datetime.strptime is both slower than this code - # and it does not support microseconds. - def process(value): - if value is not None: - return fn(*map(int, rmatch(value).groups(0))) - else: - return None - return process - class DATETIME(_DateTimeMixin, sqltypes.DateTime): - _reg = re.compile(r"(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)(?:\.(\d+))?") _storage_format = "%04d-%02d-%02d %02d:%02d:%02d.%06d" def bind_processor(self, dialect): @@ -121,10 +107,13 @@ class DATETIME(_DateTimeMixin, sqltypes.DateTime): return process def result_processor(self, dialect, coltype): - return self._result_processor(datetime.datetime) + if self._reg: + return processors.str_to_datetime_processor_factory( + self._reg, datetime.datetime) + else: + return processors.str_to_datetime class DATE(_DateTimeMixin, sqltypes.Date): - _reg = re.compile(r"(\d+)-(\d+)-(\d+)") _storage_format = "%04d-%02d-%02d" def bind_processor(self, dialect): @@ -141,10 +130,13 @@ class DATE(_DateTimeMixin, sqltypes.Date): return process def result_processor(self, dialect, coltype): - return self._result_processor(datetime.date) + if self._reg: + return processors.str_to_datetime_processor_factory( + self._reg, datetime.date) + else: + return processors.str_to_date class TIME(_DateTimeMixin, sqltypes.Time): - _reg = re.compile(r"(\d+):(\d+):(\d+)(?:\.(\d+))?") _storage_format = "%02d:%02d:%02d.%06d" def bind_processor(self, dialect): @@ -162,7 +154,11 @@ class TIME(_DateTimeMixin, sqltypes.Time): return process def result_processor(self, dialect, coltype): - return self._result_processor(datetime.time) + if self._reg: + return processors.str_to_datetime_processor_factory( + self._reg, datetime.time) + else: + return processors.str_to_time colspecs = { sqltypes.Date: DATE, diff --git a/lib/sqlalchemy/dialects/sybase/base.py b/lib/sqlalchemy/dialects/sybase/base.py index cdbf6138d..886a773d8 100644 --- a/lib/sqlalchemy/dialects/sybase/base.py +++ b/lib/sqlalchemy/dialects/sybase/base.py @@ -115,24 +115,7 @@ class SybaseUniqueIdentifier(sqltypes.TypeEngine): __visit_name__ = "UNIQUEIDENTIFIER" class SybaseBoolean(sqltypes.Boolean): - def result_processor(self, dialect, coltype): - def process(value): - if value is None: - return None - return value and True or False - return process - - def bind_processor(self, dialect): - def process(value): - if value is True: - return 1 - elif value is False: - return 0 - elif value is None: - return None - else: - return value and True or False - return process + pass class SybaseTypeCompiler(compiler.GenericTypeCompiler): def visit_large_binary(self, type_): |
