From 191fd3e27e3ef90190f8315c33ba6eb97aeaf5d2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 15:38:30 -0400 Subject: - proof of concept --- lib/sqlalchemy/orm/events.py | 9 +++++ lib/sqlalchemy/orm/persistence.py | 81 +++++++++++++++++++++------------------ lib/sqlalchemy/orm/session.py | 34 ++++++++++++++++ lib/sqlalchemy/orm/unitofwork.py | 28 +++++++++++++- 4 files changed, 113 insertions(+), 39 deletions(-) diff --git a/lib/sqlalchemy/orm/events.py b/lib/sqlalchemy/orm/events.py index aa99673ba..097726c62 100644 --- a/lib/sqlalchemy/orm/events.py +++ b/lib/sqlalchemy/orm/events.py @@ -1453,6 +1453,15 @@ class SessionEvents(event.Events): """ + def before_bulk_save(self, session, flush_context, objects): + """""" + + def after_bulk_save(self, session, flush_context, objects): + """""" + + def after_bulk_save_postexec(self, session, flush_context, objects): + """""" + def after_begin(self, session, transaction, connection): """Execute after a transaction is begun on a connection diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 9d39c39b0..511a324be 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -23,7 +23,9 @@ from ..sql import expression from . import loading -def save_obj(base_mapper, states, uowtransaction, single=False): +def save_obj( + base_mapper, states, uowtransaction, single=False, + bookkeeping=True): """Issue ``INSERT`` and/or ``UPDATE`` statements for a list of objects. @@ -43,13 +45,14 @@ def save_obj(base_mapper, states, uowtransaction, single=False): states_to_insert, states_to_update = _organize_states_for_save( base_mapper, states, - uowtransaction) + uowtransaction, bookkeeping) cached_connections = _cached_connection_dict(base_mapper) for table, mapper in base_mapper._sorted_tables.items(): insert = _collect_insert_commands(base_mapper, uowtransaction, - table, states_to_insert) + table, states_to_insert, + bookkeeping) update = _collect_update_commands(base_mapper, uowtransaction, table, states_to_update) @@ -65,7 +68,8 @@ def save_obj(base_mapper, states, uowtransaction, single=False): mapper, table, insert) _finalize_insert_update_commands(base_mapper, uowtransaction, - states_to_insert, states_to_update) + states_to_insert, states_to_update, + bookkeeping) def post_update(base_mapper, states, uowtransaction, post_update_cols): @@ -121,7 +125,8 @@ def delete_obj(base_mapper, states, uowtransaction): mapper.dispatch.after_delete(mapper, connection, state) -def _organize_states_for_save(base_mapper, states, uowtransaction): +def _organize_states_for_save( + base_mapper, states, uowtransaction, bookkeeping): """Make an initial pass across a set of states for INSERT or UPDATE. @@ -158,7 +163,7 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): # no instance_key attached to it), and another instance # with the same identity key already exists as persistent. # convert to an UPDATE if so. - if not has_identity and \ + if bookkeeping and not has_identity and \ instance_key in uowtransaction.session.identity_map: instance = \ uowtransaction.session.identity_map[instance_key] @@ -230,7 +235,7 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): def _collect_insert_commands(base_mapper, uowtransaction, table, - states_to_insert): + states_to_insert, bookkeeping): """Identify sets of values to use in INSERT statements for a list of states. @@ -261,12 +266,12 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, value = state_dict.get(prop.key, None) if value is None: - if col in pks: + if bookkeeping and col in pks: has_all_pks = False elif col.default is None and \ col.server_default is None: params[col.key] = value - elif col.server_default is not None and \ + elif bookkeeping and col.server_default is not None and \ mapper.base_mapper.eager_defaults: has_all_defaults = False @@ -756,7 +761,8 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, def _finalize_insert_update_commands(base_mapper, uowtransaction, - states_to_insert, states_to_update): + states_to_insert, states_to_update, + bookkeeping): """finalize state on states that have been inserted or updated, including calling after_insert/after_update events. @@ -765,33 +771,34 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, instance_key, row_switch in states_to_insert + \ states_to_update: - if mapper._readonly_props: - readonly = state.unmodified_intersection( - [p.key for p in mapper._readonly_props - if p.expire_on_flush or p.key not in state.dict] - ) - if readonly: - state._expire_attributes(state.dict, readonly) - - # if eager_defaults option is enabled, load - # all expired cols. Else if we have a version_id_col, make sure - # it isn't expired. - toload_now = [] - - if base_mapper.eager_defaults: - toload_now.extend(state._unloaded_non_object) - elif mapper.version_id_col is not None and \ - mapper.version_id_generator is False: - prop = mapper._columntoproperty[mapper.version_id_col] - if prop.key in state.unloaded: - toload_now.extend([prop.key]) - - if toload_now: - state.key = base_mapper._identity_key_from_state(state) - loading.load_on_ident( - uowtransaction.session.query(base_mapper), - state.key, refresh_state=state, - only_load_props=toload_now) + if bookkeeping: + if mapper._readonly_props: + readonly = state.unmodified_intersection( + [p.key for p in mapper._readonly_props + if p.expire_on_flush or p.key not in state.dict] + ) + if readonly: + state._expire_attributes(state.dict, readonly) + + # if eager_defaults option is enabled, load + # all expired cols. Else if we have a version_id_col, make sure + # it isn't expired. + toload_now = [] + + if base_mapper.eager_defaults: + toload_now.extend(state._unloaded_non_object) + elif mapper.version_id_col is not None and \ + mapper.version_id_generator is False: + prop = mapper._columntoproperty[mapper.version_id_col] + if prop.key in state.unloaded: + toload_now.extend([prop.key]) + + if toload_now: + state.key = base_mapper._identity_key_from_state(state) + loading.load_on_ident( + uowtransaction.session.query(base_mapper), + state.key, refresh_state=state, + only_load_props=toload_now) # call after_XXX extensions if not has_identity: diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 036045dba..2455c803a 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -2033,6 +2033,40 @@ class Session(_SessionClassMethods): with util.safe_reraise(): transaction.rollback(_capture_exception=True) + def bulk_save(self, objects): + self._flushing = True + flush_context = UOWTransaction(self) + + if self.dispatch.before_bulk_save: + self.dispatch.before_bulk_save( + self, flush_context, objects) + + flush_context.transaction = transaction = self.begin( + subtransactions=True) + try: + self._warn_on_events = True + try: + flush_context.bulk_save(objects) + finally: + self._warn_on_events = False + + self.dispatch.after_bulk_save( + self, flush_context, objects + ) + + flush_context.finalize_flush_changes() + + self.dispatch.after_bulk_save_postexec( + self, flush_context, objects) + + transaction.commit() + + except: + with util.safe_reraise(): + transaction.rollback(_capture_exception=True) + finally: + self._flushing = False + def is_modified(self, instance, include_collections=True, passive=True): """Return ``True`` if the given instance has locally diff --git a/lib/sqlalchemy/orm/unitofwork.py b/lib/sqlalchemy/orm/unitofwork.py index 71e61827b..8df24e95a 100644 --- a/lib/sqlalchemy/orm/unitofwork.py +++ b/lib/sqlalchemy/orm/unitofwork.py @@ -16,6 +16,7 @@ organizes them in order of dependency, and executes. from .. import util, event from ..util import topological from . import attributes, persistence, util as orm_util +import itertools def track_cascade_events(descriptor, prop): @@ -379,14 +380,37 @@ class UOWTransaction(object): execute() method has succeeded and the transaction has been committed. """ + if not self.states: + return + states = set(self.states) isdel = set( s for (s, (isdelete, listonly)) in self.states.items() if isdelete ) other = states.difference(isdel) - self.session._remove_newly_deleted(isdel) - self.session._register_newly_persistent(other) + if isdel: + self.session._remove_newly_deleted(isdel) + if other: + self.session._register_newly_persistent(other) + + def bulk_save(self, objects): + for (base_mapper, in_session), states in itertools.groupby( + (attributes.instance_state(obj) for obj in objects), + lambda state: + ( + state.mapper.base_mapper, + state.key is self.session.hash_key + )): + + persistence.save_obj( + base_mapper, list(states), self, bookkeeping=in_session) + + if in_session: + self.states.update( + (state, (False, False)) + for state in states + ) class IterateMappersMixin(object): -- cgit v1.2.1 From 6bc676f56d57d5ea4dc298f63d0e3a77c0f4a4a1 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 17:44:58 -0400 Subject: dev --- doc/build/faq.rst | 81 +++++++++++++++++++++++++++++++-------- lib/sqlalchemy/orm/persistence.py | 59 +++++++++++++++------------- lib/sqlalchemy/orm/session.py | 23 ++++++++--- lib/sqlalchemy/orm/state.py | 15 ++++++++ lib/sqlalchemy/orm/unitofwork.py | 10 ++--- 5 files changed, 135 insertions(+), 53 deletions(-) diff --git a/doc/build/faq.rst b/doc/build/faq.rst index 3dc81026b..b777f908f 100644 --- a/doc/build/faq.rst +++ b/doc/build/faq.rst @@ -907,10 +907,12 @@ methods of inserting rows, going from the most automated to the least. With cPython 2.7, runtimes observed:: classics-MacBook-Pro:sqlalchemy classic$ python test.py - SQLAlchemy ORM: Total time for 100000 records 14.3528850079 secs - SQLAlchemy ORM pk given: Total time for 100000 records 10.0164160728 secs - SQLAlchemy Core: Total time for 100000 records 0.775382995605 secs - sqlite3: Total time for 100000 records 0.676795005798 sec + SQLAlchemy ORM: Total time for 100000 records 12.4703581333 secs + SQLAlchemy ORM pk given: Total time for 100000 records 7.32723999023 secs + SQLAlchemy ORM bulk_save_objects(): Total time for 100000 records 3.43464708328 secs + SQLAlchemy ORM bulk_save_mappings(): Total time for 100000 records 2.37040805817 secs + SQLAlchemy Core: Total time for 100000 records 0.495043992996 secs + sqlite3: Total time for 100000 records 0.508063077927 sec We can reduce the time by a factor of three using recent versions of `Pypy `_:: @@ -933,11 +935,13 @@ Script:: DBSession = scoped_session(sessionmaker()) engine = None + class Customer(Base): __tablename__ = "customer" id = Column(Integer, primary_key=True) name = Column(String(255)) + def init_sqlalchemy(dbname='sqlite:///sqlalchemy.db'): global engine engine = create_engine(dbname, echo=False) @@ -946,69 +950,114 @@ Script:: Base.metadata.drop_all(engine) Base.metadata.create_all(engine) + def test_sqlalchemy_orm(n=100000): init_sqlalchemy() t0 = time.time() - for i in range(n): + for i in xrange(n): customer = Customer() customer.name = 'NAME ' + str(i) DBSession.add(customer) if i % 1000 == 0: DBSession.flush() DBSession.commit() - print("SQLAlchemy ORM: Total time for " + str(n) + - " records " + str(time.time() - t0) + " secs") + print( + "SQLAlchemy ORM: Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + def test_sqlalchemy_orm_pk_given(n=100000): init_sqlalchemy() t0 = time.time() - for i in range(n): + for i in xrange(n): customer = Customer(id=i+1, name="NAME " + str(i)) DBSession.add(customer) if i % 1000 == 0: DBSession.flush() DBSession.commit() - print("SQLAlchemy ORM pk given: Total time for " + str(n) + + print( + "SQLAlchemy ORM pk given: Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + + + def test_sqlalchemy_orm_bulk_save(n=100000): + init_sqlalchemy() + t0 = time.time() + n1 = n + while n1 > 0: + n1 = n1 - 10000 + DBSession.bulk_save_objects( + [ + Customer(name="NAME " + str(i)) + for i in xrange(min(10000, n1)) + ] + ) + DBSession.commit() + print( + "SQLAlchemy ORM bulk_save_objects(): Total time for " + str(n) + " records " + str(time.time() - t0) + " secs") + + def test_sqlalchemy_orm_bulk_save_mappings(n=100000): + init_sqlalchemy() + t0 = time.time() + DBSession.bulk_save_mappings( + Customer, + [ + dict(name="NAME " + str(i)) + for i in xrange(n) + ] + ) + DBSession.commit() + print( + "SQLAlchemy ORM bulk_save_mappings(): Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + + def test_sqlalchemy_core(n=100000): init_sqlalchemy() t0 = time.time() engine.execute( Customer.__table__.insert(), - [{"name": 'NAME ' + str(i)} for i in range(n)] + [{"name": 'NAME ' + str(i)} for i in xrange(n)] ) - print("SQLAlchemy Core: Total time for " + str(n) + + print( + "SQLAlchemy Core: Total time for " + str(n) + " records " + str(time.time() - t0) + " secs") + def init_sqlite3(dbname): conn = sqlite3.connect(dbname) c = conn.cursor() c.execute("DROP TABLE IF EXISTS customer") - c.execute("CREATE TABLE customer (id INTEGER NOT NULL, " - "name VARCHAR(255), PRIMARY KEY(id))") + c.execute( + "CREATE TABLE customer (id INTEGER NOT NULL, " + "name VARCHAR(255), PRIMARY KEY(id))") conn.commit() return conn + def test_sqlite3(n=100000, dbname='sqlite3.db'): conn = init_sqlite3(dbname) c = conn.cursor() t0 = time.time() - for i in range(n): + for i in xrange(n): row = ('NAME ' + str(i),) c.execute("INSERT INTO customer (name) VALUES (?)", row) conn.commit() - print("sqlite3: Total time for " + str(n) + + print( + "sqlite3: Total time for " + str(n) + " records " + str(time.time() - t0) + " sec") if __name__ == '__main__': test_sqlalchemy_orm(100000) test_sqlalchemy_orm_pk_given(100000) + test_sqlalchemy_orm_bulk_save(100000) + test_sqlalchemy_orm_bulk_save_mappings(100000) test_sqlalchemy_core(100000) test_sqlite3(100000) - Sessions / Queries =================== diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 511a324be..64c8440c4 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -18,7 +18,7 @@ import operator from itertools import groupby from .. import sql, util, exc as sa_exc, schema from . import attributes, sync, exc as orm_exc, evaluator -from .base import _state_mapper, state_str, _attr_as_key +from .base import state_str, _attr_as_key from ..sql import expression from . import loading @@ -65,7 +65,8 @@ def save_obj( if insert: _emit_insert_statements(base_mapper, uowtransaction, cached_connections, - mapper, table, insert) + mapper, table, insert, + bookkeeping) _finalize_insert_update_commands(base_mapper, uowtransaction, states_to_insert, states_to_update, @@ -140,13 +141,16 @@ def _organize_states_for_save( states_to_insert = [] states_to_update = [] + instance_key = None for state, dict_, mapper, connection in _connections_for_states( base_mapper, uowtransaction, states): has_identity = bool(state.key) - instance_key = state.key or mapper._identity_key_from_state(state) + + if bookkeeping: + instance_key = state.key or mapper._identity_key_from_state(state) row_switch = None @@ -188,12 +192,12 @@ def _organize_states_for_save( if not has_identity and not row_switch: states_to_insert.append( (state, dict_, mapper, connection, - has_identity, instance_key, row_switch) + has_identity, row_switch) ) else: states_to_update.append( (state, dict_, mapper, connection, - has_identity, instance_key, row_switch) + has_identity, row_switch) ) return states_to_insert, states_to_update @@ -242,7 +246,8 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, """ insert = [] for state, state_dict, mapper, connection, has_identity, \ - instance_key, row_switch in states_to_insert: + row_switch in states_to_insert: + if table not in mapper._pks_by_table: continue @@ -265,13 +270,13 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, prop = mapper._columntoproperty[col] value = state_dict.get(prop.key, None) - if value is None: - if bookkeeping and col in pks: + if bookkeeping and value is None: + if col in pks: has_all_pks = False elif col.default is None and \ col.server_default is None: params[col.key] = value - elif bookkeeping and col.server_default is not None and \ + elif col.server_default is not None and \ mapper.base_mapper.eager_defaults: has_all_defaults = False @@ -301,7 +306,7 @@ def _collect_update_commands(base_mapper, uowtransaction, update = [] for state, state_dict, mapper, connection, has_identity, \ - instance_key, row_switch in states_to_update: + row_switch in states_to_update: if table not in mapper._pks_by_table: continue @@ -567,7 +572,8 @@ def _emit_update_statements(base_mapper, uowtransaction, def _emit_insert_statements(base_mapper, uowtransaction, - cached_connections, mapper, table, insert): + cached_connections, mapper, table, insert, + bookkeeping): """Emit INSERT statements corresponding to value lists collected by _collect_insert_commands().""" @@ -593,19 +599,20 @@ def _emit_insert_statements(base_mapper, uowtransaction, c = cached_connections[connection].\ execute(statement, multiparams) - for (state, state_dict, params, mapper_rec, - conn, value_params, has_all_pks, has_all_defaults), \ - last_inserted_params in \ - zip(records, c.context.compiled_parameters): - _postfetch( - mapper_rec, - uowtransaction, - table, - state, - state_dict, - c, - last_inserted_params, - value_params) + if bookkeeping: + for (state, state_dict, params, mapper_rec, + conn, value_params, has_all_pks, has_all_defaults), \ + last_inserted_params in \ + zip(records, c.context.compiled_parameters): + _postfetch( + mapper_rec, + uowtransaction, + table, + state, + state_dict, + c, + last_inserted_params, + value_params) else: if not has_all_defaults and base_mapper.eager_defaults: @@ -768,7 +775,7 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, """ for state, state_dict, mapper, connection, has_identity, \ - instance_key, row_switch in states_to_insert + \ + row_switch in states_to_insert + \ states_to_update: if bookkeeping: @@ -871,7 +878,7 @@ def _connections_for_states(base_mapper, uowtransaction, states): if connection_callable: connection = connection_callable(base_mapper, state.obj()) - mapper = _state_mapper(state) + mapper = state.manager.mapper yield state, state.dict, mapper, connection diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 2455c803a..546355611 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -482,7 +482,7 @@ class Session(_SessionClassMethods): '__contains__', '__iter__', 'add', 'add_all', 'begin', 'begin_nested', 'close', 'commit', 'connection', 'delete', 'execute', 'expire', 'expire_all', 'expunge', 'expunge_all', 'flush', 'get_bind', - 'is_modified', + 'is_modified', 'bulk_save_objects', 'bulk_save_mappings', 'merge', 'query', 'refresh', 'rollback', 'scalar') @@ -2033,31 +2033,42 @@ class Session(_SessionClassMethods): with util.safe_reraise(): transaction.rollback(_capture_exception=True) - def bulk_save(self, objects): + def bulk_save_objects(self, objects): + self._bulk_save((attributes.instance_state(obj) for obj in objects)) + + def bulk_save_mappings(self, mapper, mappings): + mapper = class_mapper(mapper) + + self._bulk_save(( + statelib.MappingState(mapper, mapping) + for mapping in mappings) + ) + + def _bulk_save(self, states): self._flushing = True flush_context = UOWTransaction(self) if self.dispatch.before_bulk_save: self.dispatch.before_bulk_save( - self, flush_context, objects) + self, flush_context, states) flush_context.transaction = transaction = self.begin( subtransactions=True) try: self._warn_on_events = True try: - flush_context.bulk_save(objects) + flush_context.bulk_save(states) finally: self._warn_on_events = False self.dispatch.after_bulk_save( - self, flush_context, objects + self, flush_context, states ) flush_context.finalize_flush_changes() self.dispatch.after_bulk_save_postexec( - self, flush_context, objects) + self, flush_context, states) transaction.commit() diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py index fe8ccd222..e941bc1a4 100644 --- a/lib/sqlalchemy/orm/state.py +++ b/lib/sqlalchemy/orm/state.py @@ -580,6 +580,21 @@ class InstanceState(interfaces.InspectionAttr): state._strong_obj = None +class MappingState(InstanceState): + committed_state = {} + callables = {} + + def __init__(self, mapper, mapping): + self.class_ = mapper.class_ + self.manager = mapper.class_manager + self.modified = True + self._dict = mapping + + @property + def dict(self): + return self._dict + + class AttributeState(object): """Provide an inspection interface corresponding to a particular attribute on a particular mapped object. diff --git a/lib/sqlalchemy/orm/unitofwork.py b/lib/sqlalchemy/orm/unitofwork.py index 8df24e95a..bc8a0f556 100644 --- a/lib/sqlalchemy/orm/unitofwork.py +++ b/lib/sqlalchemy/orm/unitofwork.py @@ -394,9 +394,9 @@ class UOWTransaction(object): if other: self.session._register_newly_persistent(other) - def bulk_save(self, objects): - for (base_mapper, in_session), states in itertools.groupby( - (attributes.instance_state(obj) for obj in objects), + def bulk_save(self, states): + for (base_mapper, in_session), states_ in itertools.groupby( + states, lambda state: ( state.mapper.base_mapper, @@ -404,12 +404,12 @@ class UOWTransaction(object): )): persistence.save_obj( - base_mapper, list(states), self, bookkeeping=in_session) + base_mapper, list(states_), self, bookkeeping=in_session) if in_session: self.states.update( (state, (False, False)) - for state in states + for state in states_ ) -- cgit v1.2.1 From 591f2e4ed2d455cb2c5b9ece43d79fde4b109510 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 19:47:23 -0400 Subject: - change to be represented as two very fast bulk_insert() and bulk_update() methods --- doc/build/faq.rst | 37 ++---- lib/sqlalchemy/orm/events.py | 9 +- lib/sqlalchemy/orm/persistence.py | 255 ++++++++++++++++++++++++++------------ lib/sqlalchemy/orm/session.py | 57 ++++----- lib/sqlalchemy/orm/state.py | 15 --- lib/sqlalchemy/orm/unitofwork.py | 22 +--- 6 files changed, 223 insertions(+), 172 deletions(-) diff --git a/doc/build/faq.rst b/doc/build/faq.rst index b777f908f..487f5b953 100644 --- a/doc/build/faq.rst +++ b/doc/build/faq.rst @@ -907,12 +907,11 @@ methods of inserting rows, going from the most automated to the least. With cPython 2.7, runtimes observed:: classics-MacBook-Pro:sqlalchemy classic$ python test.py - SQLAlchemy ORM: Total time for 100000 records 12.4703581333 secs - SQLAlchemy ORM pk given: Total time for 100000 records 7.32723999023 secs - SQLAlchemy ORM bulk_save_objects(): Total time for 100000 records 3.43464708328 secs - SQLAlchemy ORM bulk_save_mappings(): Total time for 100000 records 2.37040805817 secs - SQLAlchemy Core: Total time for 100000 records 0.495043992996 secs - sqlite3: Total time for 100000 records 0.508063077927 sec + SQLAlchemy ORM: Total time for 100000 records 12.0471920967 secs + SQLAlchemy ORM pk given: Total time for 100000 records 7.06283402443 secs + SQLAlchemy ORM bulk_save_objects(): Total time for 100000 records 0.856323003769 secs + SQLAlchemy Core: Total time for 100000 records 0.485800027847 secs + sqlite3: Total time for 100000 records 0.487842082977 sec We can reduce the time by a factor of three using recent versions of `Pypy `_:: @@ -980,15 +979,16 @@ Script:: " records " + str(time.time() - t0) + " secs") - def test_sqlalchemy_orm_bulk_save(n=100000): + def test_sqlalchemy_orm_bulk_insert(n=100000): init_sqlalchemy() t0 = time.time() n1 = n while n1 > 0: n1 = n1 - 10000 - DBSession.bulk_save_objects( + DBSession.bulk_insert_mappings( + Customer, [ - Customer(name="NAME " + str(i)) + dict(name="NAME " + str(i)) for i in xrange(min(10000, n1)) ] ) @@ -998,22 +998,6 @@ Script:: " records " + str(time.time() - t0) + " secs") - def test_sqlalchemy_orm_bulk_save_mappings(n=100000): - init_sqlalchemy() - t0 = time.time() - DBSession.bulk_save_mappings( - Customer, - [ - dict(name="NAME " + str(i)) - for i in xrange(n) - ] - ) - DBSession.commit() - print( - "SQLAlchemy ORM bulk_save_mappings(): Total time for " + str(n) + - " records " + str(time.time() - t0) + " secs") - - def test_sqlalchemy_core(n=100000): init_sqlalchemy() t0 = time.time() @@ -1052,8 +1036,7 @@ Script:: if __name__ == '__main__': test_sqlalchemy_orm(100000) test_sqlalchemy_orm_pk_given(100000) - test_sqlalchemy_orm_bulk_save(100000) - test_sqlalchemy_orm_bulk_save_mappings(100000) + test_sqlalchemy_orm_bulk_insert(100000) test_sqlalchemy_core(100000) test_sqlite3(100000) diff --git a/lib/sqlalchemy/orm/events.py b/lib/sqlalchemy/orm/events.py index 097726c62..37ea3071b 100644 --- a/lib/sqlalchemy/orm/events.py +++ b/lib/sqlalchemy/orm/events.py @@ -1453,13 +1453,16 @@ class SessionEvents(event.Events): """ - def before_bulk_save(self, session, flush_context, objects): + def before_bulk_insert(self, session, flush_context, mapper, mappings): """""" - def after_bulk_save(self, session, flush_context, objects): + def after_bulk_insert(self, session, flush_context, mapper, mappings): """""" - def after_bulk_save_postexec(self, session, flush_context, objects): + def before_bulk_update(self, session, flush_context, mapper, mappings): + """""" + + def after_bulk_update(self, session, flush_context, mapper, mappings): """""" def after_begin(self, session, transaction, connection): diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 64c8440c4..a8d4bd695 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -23,9 +23,104 @@ from ..sql import expression from . import loading +def bulk_insert(mapper, mappings, uowtransaction): + base_mapper = mapper.base_mapper + + cached_connections = _cached_connection_dict(base_mapper) + + if uowtransaction.session.connection_callable: + raise NotImplementedError( + "connection_callable / per-instance sharding " + "not supported in bulk_insert()") + + connection = uowtransaction.transaction.connection(base_mapper) + + for table, sub_mapper in base_mapper._sorted_tables.items(): + if not mapper.isa(sub_mapper): + continue + + to_translate = dict( + (mapper._columntoproperty[col].key, col.key) + for col in mapper._cols_by_table[table] + ) + has_version_generator = mapper.version_id_generator is not False and \ + mapper.version_id_col is not None + multiparams = [] + for mapping in mappings: + params = dict( + (k, mapping.get(v)) for k, v in to_translate.items() + ) + if has_version_generator: + params[mapper.version_id_col.key] = \ + mapper.version_id_generator(None) + multiparams.append(params) + + statement = base_mapper._memo(('insert', table), table.insert) + cached_connections[connection].execute(statement, multiparams) + + +def bulk_update(mapper, mappings, uowtransaction): + base_mapper = mapper.base_mapper + + cached_connections = _cached_connection_dict(base_mapper) + + if uowtransaction.session.connection_callable: + raise NotImplementedError( + "connection_callable / per-instance sharding " + "not supported in bulk_update()") + + connection = uowtransaction.transaction.connection(base_mapper) + + for table, sub_mapper in base_mapper._sorted_tables.items(): + if not mapper.isa(sub_mapper): + continue + + needs_version_id = sub_mapper.version_id_col is not None and \ + table.c.contains_column(sub_mapper.version_id_col) + + def update_stmt(): + return _update_stmt_for_mapper(sub_mapper, table, needs_version_id) + + statement = base_mapper._memo(('update', table), update_stmt) + + pks = mapper._pks_by_table[table] + to_translate = dict( + (mapper._columntoproperty[col].key, col._label + if col in pks else col.key) + for col in mapper._cols_by_table[table] + ) + + for colnames, sub_mappings in groupby( + mappings, + lambda mapping: sorted(tuple(mapping.keys()))): + + multiparams = [] + for mapping in sub_mappings: + params = dict( + (to_translate[k], v) for k, v in mapping.items() + ) + multiparams.append(params) + + c = cached_connections[connection].execute(statement, multiparams) + + rows = c.rowcount + + if connection.dialect.supports_sane_rowcount: + if rows != len(multiparams): + raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." % + (table.description, len(multiparams), rows)) + + elif needs_version_id: + util.warn("Dialect %s does not support updated rowcount " + "- versioning cannot be verified." % + c.dialect.dialect_description, + stacklevel=12) + + def save_obj( - base_mapper, states, uowtransaction, single=False, - bookkeeping=True): + base_mapper, states, uowtransaction, single=False): """Issue ``INSERT`` and/or ``UPDATE`` statements for a list of objects. @@ -45,14 +140,13 @@ def save_obj( states_to_insert, states_to_update = _organize_states_for_save( base_mapper, states, - uowtransaction, bookkeeping) + uowtransaction) cached_connections = _cached_connection_dict(base_mapper) for table, mapper in base_mapper._sorted_tables.items(): insert = _collect_insert_commands(base_mapper, uowtransaction, - table, states_to_insert, - bookkeeping) + table, states_to_insert) update = _collect_update_commands(base_mapper, uowtransaction, table, states_to_update) @@ -65,12 +159,11 @@ def save_obj( if insert: _emit_insert_statements(base_mapper, uowtransaction, cached_connections, - mapper, table, insert, - bookkeeping) + mapper, table, insert) - _finalize_insert_update_commands(base_mapper, uowtransaction, - states_to_insert, states_to_update, - bookkeeping) + _finalize_insert_update_commands( + base_mapper, uowtransaction, + states_to_insert, states_to_update) def post_update(base_mapper, states, uowtransaction, post_update_cols): @@ -126,8 +219,7 @@ def delete_obj(base_mapper, states, uowtransaction): mapper.dispatch.after_delete(mapper, connection, state) -def _organize_states_for_save( - base_mapper, states, uowtransaction, bookkeeping): +def _organize_states_for_save(base_mapper, states, uowtransaction): """Make an initial pass across a set of states for INSERT or UPDATE. @@ -149,8 +241,7 @@ def _organize_states_for_save( has_identity = bool(state.key) - if bookkeeping: - instance_key = state.key or mapper._identity_key_from_state(state) + instance_key = state.key or mapper._identity_key_from_state(state) row_switch = None @@ -167,7 +258,7 @@ def _organize_states_for_save( # no instance_key attached to it), and another instance # with the same identity key already exists as persistent. # convert to an UPDATE if so. - if bookkeeping and not has_identity and \ + if not has_identity and \ instance_key in uowtransaction.session.identity_map: instance = \ uowtransaction.session.identity_map[instance_key] @@ -239,7 +330,7 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): def _collect_insert_commands(base_mapper, uowtransaction, table, - states_to_insert, bookkeeping): + states_to_insert): """Identify sets of values to use in INSERT statements for a list of states. @@ -270,7 +361,7 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, prop = mapper._columntoproperty[col] value = state_dict.get(prop.key, None) - if bookkeeping and value is None: + if value is None: if col in pks: has_all_pks = False elif col.default is None and \ @@ -481,6 +572,28 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, return delete +def _update_stmt_for_mapper(mapper, table, needs_version_id): + clause = sql.and_() + + for col in mapper._pks_by_table[table]: + clause.clauses.append(col == sql.bindparam(col._label, + type_=col.type)) + + if needs_version_id: + clause.clauses.append( + mapper.version_id_col == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type)) + + stmt = table.update(clause) + if mapper.base_mapper.eager_defaults: + stmt = stmt.return_defaults() + elif mapper.version_id_col is not None: + stmt = stmt.return_defaults(mapper.version_id_col) + + return stmt + + def _emit_update_statements(base_mapper, uowtransaction, cached_connections, mapper, table, update): """Emit UPDATE statements corresponding to value lists collected @@ -490,25 +603,7 @@ def _emit_update_statements(base_mapper, uowtransaction, table.c.contains_column(mapper.version_id_col) def update_stmt(): - clause = sql.and_() - - for col in mapper._pks_by_table[table]: - clause.clauses.append(col == sql.bindparam(col._label, - type_=col.type)) - - if needs_version_id: - clause.clauses.append( - mapper.version_id_col == sql.bindparam( - mapper.version_id_col._label, - type_=mapper.version_id_col.type)) - - stmt = table.update(clause) - if mapper.base_mapper.eager_defaults: - stmt = stmt.return_defaults() - elif mapper.version_id_col is not None: - stmt = stmt.return_defaults(mapper.version_id_col) - - return stmt + return _update_stmt_for_mapper(mapper, table, needs_version_id) statement = base_mapper._memo(('update', table), update_stmt) @@ -572,8 +667,7 @@ def _emit_update_statements(base_mapper, uowtransaction, def _emit_insert_statements(base_mapper, uowtransaction, - cached_connections, mapper, table, insert, - bookkeeping): + cached_connections, mapper, table, insert): """Emit INSERT statements corresponding to value lists collected by _collect_insert_commands().""" @@ -599,20 +693,19 @@ def _emit_insert_statements(base_mapper, uowtransaction, c = cached_connections[connection].\ execute(statement, multiparams) - if bookkeeping: - for (state, state_dict, params, mapper_rec, - conn, value_params, has_all_pks, has_all_defaults), \ - last_inserted_params in \ - zip(records, c.context.compiled_parameters): - _postfetch( - mapper_rec, - uowtransaction, - table, - state, - state_dict, - c, - last_inserted_params, - value_params) + for (state, state_dict, params, mapper_rec, + conn, value_params, has_all_pks, has_all_defaults), \ + last_inserted_params in \ + zip(records, c.context.compiled_parameters): + _postfetch( + mapper_rec, + uowtransaction, + table, + state, + state_dict, + c, + last_inserted_params, + value_params) else: if not has_all_defaults and base_mapper.eager_defaults: @@ -768,8 +861,7 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, def _finalize_insert_update_commands(base_mapper, uowtransaction, - states_to_insert, states_to_update, - bookkeeping): + states_to_insert, states_to_update): """finalize state on states that have been inserted or updated, including calling after_insert/after_update events. @@ -778,34 +870,33 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, row_switch in states_to_insert + \ states_to_update: - if bookkeeping: - if mapper._readonly_props: - readonly = state.unmodified_intersection( - [p.key for p in mapper._readonly_props - if p.expire_on_flush or p.key not in state.dict] - ) - if readonly: - state._expire_attributes(state.dict, readonly) - - # if eager_defaults option is enabled, load - # all expired cols. Else if we have a version_id_col, make sure - # it isn't expired. - toload_now = [] - - if base_mapper.eager_defaults: - toload_now.extend(state._unloaded_non_object) - elif mapper.version_id_col is not None and \ - mapper.version_id_generator is False: - prop = mapper._columntoproperty[mapper.version_id_col] - if prop.key in state.unloaded: - toload_now.extend([prop.key]) - - if toload_now: - state.key = base_mapper._identity_key_from_state(state) - loading.load_on_ident( - uowtransaction.session.query(base_mapper), - state.key, refresh_state=state, - only_load_props=toload_now) + if mapper._readonly_props: + readonly = state.unmodified_intersection( + [p.key for p in mapper._readonly_props + if p.expire_on_flush or p.key not in state.dict] + ) + if readonly: + state._expire_attributes(state.dict, readonly) + + # if eager_defaults option is enabled, load + # all expired cols. Else if we have a version_id_col, make sure + # it isn't expired. + toload_now = [] + + if base_mapper.eager_defaults: + toload_now.extend(state._unloaded_non_object) + elif mapper.version_id_col is not None and \ + mapper.version_id_generator is False: + prop = mapper._columntoproperty[mapper.version_id_col] + if prop.key in state.unloaded: + toload_now.extend([prop.key]) + + if toload_now: + state.key = base_mapper._identity_key_from_state(state) + loading.load_on_ident( + uowtransaction.session.query(base_mapper), + state.key, refresh_state=state, + only_load_props=toload_now) # call after_XXX extensions if not has_identity: diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 546355611..3199a4332 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -20,6 +20,7 @@ from .base import ( _class_to_mapper, _state_mapper, object_state, _none_set, state_str, instance_str ) +import itertools from .unitofwork import UOWTransaction from . import state as statelib import sys @@ -482,7 +483,8 @@ class Session(_SessionClassMethods): '__contains__', '__iter__', 'add', 'add_all', 'begin', 'begin_nested', 'close', 'commit', 'connection', 'delete', 'execute', 'expire', 'expire_all', 'expunge', 'expunge_all', 'flush', 'get_bind', - 'is_modified', 'bulk_save_objects', 'bulk_save_mappings', + 'is_modified', 'bulk_save_objects', 'bulk_insert_mappings', + 'bulk_update_mappings', 'merge', 'query', 'refresh', 'rollback', 'scalar') @@ -2034,42 +2036,41 @@ class Session(_SessionClassMethods): transaction.rollback(_capture_exception=True) def bulk_save_objects(self, objects): - self._bulk_save((attributes.instance_state(obj) for obj in objects)) + for (mapper, isupdate), states in itertools.groupby( + (attributes.instance_state(obj) for obj in objects), + lambda state: (state.mapper, state.key is not None) + ): + if isupdate: + self.bulk_update_mappings(mapper, (s.dict for s in states)) + else: + self.bulk_insert_mappings(mapper, (s.dict for s in states)) - def bulk_save_mappings(self, mapper, mappings): - mapper = class_mapper(mapper) + def bulk_insert_mappings(self, mapper, mappings): + self._bulk_save_mappings(mapper, mappings, False) - self._bulk_save(( - statelib.MappingState(mapper, mapping) - for mapping in mappings) - ) + def bulk_update_mappings(self, mapper, mappings): + self._bulk_save_mappings(mapper, mappings, True) - def _bulk_save(self, states): + def _bulk_save_mappings(self, mapper, mappings, isupdate): + mapper = _class_to_mapper(mapper) self._flushing = True flush_context = UOWTransaction(self) - if self.dispatch.before_bulk_save: - self.dispatch.before_bulk_save( - self, flush_context, states) - flush_context.transaction = transaction = self.begin( subtransactions=True) try: - self._warn_on_events = True - try: - flush_context.bulk_save(states) - finally: - self._warn_on_events = False - - self.dispatch.after_bulk_save( - self, flush_context, states - ) - - flush_context.finalize_flush_changes() - - self.dispatch.after_bulk_save_postexec( - self, flush_context, states) - + if isupdate: + self.dispatch.before_bulk_update( + self, flush_context, mapper, mappings) + flush_context.bulk_update(mapper, mappings) + self.dispatch.after_bulk_update( + self, flush_context, mapper, mappings) + else: + self.dispatch.before_bulk_insert( + self, flush_context, mapper, mappings) + flush_context.bulk_insert(mapper, mappings) + self.dispatch.after_bulk_insert( + self, flush_context, mapper, mappings) transaction.commit() except: diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py index e941bc1a4..fe8ccd222 100644 --- a/lib/sqlalchemy/orm/state.py +++ b/lib/sqlalchemy/orm/state.py @@ -580,21 +580,6 @@ class InstanceState(interfaces.InspectionAttr): state._strong_obj = None -class MappingState(InstanceState): - committed_state = {} - callables = {} - - def __init__(self, mapper, mapping): - self.class_ = mapper.class_ - self.manager = mapper.class_manager - self.modified = True - self._dict = mapping - - @property - def dict(self): - return self._dict - - class AttributeState(object): """Provide an inspection interface corresponding to a particular attribute on a particular mapped object. diff --git a/lib/sqlalchemy/orm/unitofwork.py b/lib/sqlalchemy/orm/unitofwork.py index bc8a0f556..b3a1519c5 100644 --- a/lib/sqlalchemy/orm/unitofwork.py +++ b/lib/sqlalchemy/orm/unitofwork.py @@ -394,23 +394,11 @@ class UOWTransaction(object): if other: self.session._register_newly_persistent(other) - def bulk_save(self, states): - for (base_mapper, in_session), states_ in itertools.groupby( - states, - lambda state: - ( - state.mapper.base_mapper, - state.key is self.session.hash_key - )): - - persistence.save_obj( - base_mapper, list(states_), self, bookkeeping=in_session) - - if in_session: - self.states.update( - (state, (False, False)) - for state in states_ - ) + def bulk_insert(self, mapper, mappings): + persistence.bulk_insert(mapper, mappings, self) + + def bulk_update(self, mapper, mappings): + persistence.bulk_update(mapper, mappings, self) class IterateMappersMixin(object): -- cgit v1.2.1 From 8773307257550e86801217f2b77d47047718807a Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 18:22:08 -0400 Subject: - refine this enough so that _collect_insert_commands() seems to be more than twice as fast now (.039 vs. .091); bulk_insert() and bulk_update() do their own collection but now both call into _emit_insert_statements() / _emit_update_statements(); the approach seems to have no impact on insert speed, still .85 for the insert test --- lib/sqlalchemy/orm/mapper.py | 35 ++++++ lib/sqlalchemy/orm/persistence.py | 259 +++++++++++++++++++------------------- 2 files changed, 161 insertions(+), 133 deletions(-) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 06ec2bf14..fc15769cd 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1892,6 +1892,41 @@ class Mapper(InspectionAttr): """ + @_memoized_configured_property + def _col_to_propkey(self): + return dict( + ( + table, + [ + (col, self._columntoproperty[col].key) + for col in columns + ] + ) + for table, columns in self._cols_by_table.items() + ) + + @_memoized_configured_property + def _pk_keys_by_table(self): + return dict( + ( + table, + frozenset([col.key for col in pks]) + ) + for table, pks in self._pks_by_table.items() + ) + + @_memoized_configured_property + def _server_default_cols(self): + return dict( + ( + table, + frozenset([ + col for col in columns + if col.server_default is not None]) + ) + for table, columns in self._cols_by_table.items() + ) + @property def selectable(self): """The :func:`.select` construct this :class:`.Mapper` selects from diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index a8d4bd695..782d94dc8 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -34,29 +34,35 @@ def bulk_insert(mapper, mappings, uowtransaction): "not supported in bulk_insert()") connection = uowtransaction.transaction.connection(base_mapper) - + value_params = {} for table, sub_mapper in base_mapper._sorted_tables.items(): if not mapper.isa(sub_mapper): continue - to_translate = dict( - (mapper._columntoproperty[col].key, col.key) - for col in mapper._cols_by_table[table] - ) has_version_generator = mapper.version_id_generator is not False and \ mapper.version_id_col is not None - multiparams = [] + + records = [] for mapping in mappings: params = dict( - (k, mapping.get(v)) for k, v in to_translate.items() + (col.key, mapping[propkey]) + for col, propkey in mapper._col_to_propkey[table] + if propkey in mapping ) + if has_version_generator: params[mapper.version_id_col.key] = \ mapper.version_id_generator(None) - multiparams.append(params) - statement = base_mapper._memo(('insert', table), table.insert) - cached_connections[connection].execute(statement, multiparams) + records.append( + (None, None, params, sub_mapper, + connection, value_params, True, True) + ) + + _emit_insert_statements(base_mapper, uowtransaction, + cached_connections, + mapper, table, records, + bookkeeping=False) def bulk_update(mapper, mappings, uowtransaction): @@ -71,52 +77,41 @@ def bulk_update(mapper, mappings, uowtransaction): connection = uowtransaction.transaction.connection(base_mapper) + value_params = {} for table, sub_mapper in base_mapper._sorted_tables.items(): if not mapper.isa(sub_mapper): continue - needs_version_id = sub_mapper.version_id_col is not None and \ - table.c.contains_column(sub_mapper.version_id_col) - - def update_stmt(): - return _update_stmt_for_mapper(sub_mapper, table, needs_version_id) - - statement = base_mapper._memo(('update', table), update_stmt) + label_pks = mapper._pks_by_table[table] + if mapper.version_id_col is not None: + label_pks = label_pks.union([mapper.version_id_col]) - pks = mapper._pks_by_table[table] to_translate = dict( - (mapper._columntoproperty[col].key, col._label - if col in pks else col.key) - for col in mapper._cols_by_table[table] + (propkey, col._label if col in label_pks else col.key) + for col, propkey in mapper._col_to_propkey[table] ) - for colnames, sub_mappings in groupby( - mappings, - lambda mapping: sorted(tuple(mapping.keys()))): - - multiparams = [] - for mapping in sub_mappings: - params = dict( - (to_translate[k], v) for k, v in mapping.items() - ) - multiparams.append(params) - - c = cached_connections[connection].execute(statement, multiparams) + records = [] + for mapping in mappings: + params = dict( + (to_translate[k], v) for k, v in mapping.items() + ) - rows = c.rowcount + if mapper.version_id_generator is not False and \ + mapper.version_id_col is not None and \ + mapper.version_id_col.key not in params: + params[mapper.version_id_col.key] = \ + mapper.version_id_generator( + params[mapper.version_id_col._label]) - if connection.dialect.supports_sane_rowcount: - if rows != len(multiparams): - raise orm_exc.StaleDataError( - "UPDATE statement on table '%s' expected to " - "update %d row(s); %d were matched." % - (table.description, len(multiparams), rows)) + records.append( + (None, None, params, sub_mapper, connection, value_params) + ) - elif needs_version_id: - util.warn("Dialect %s does not support updated rowcount " - "- versioning cannot be verified." % - c.dialect.dialect_description, - stacklevel=12) + _emit_update_statements(base_mapper, uowtransaction, + cached_connections, + mapper, table, records, + bookkeeping=False) def save_obj( @@ -342,39 +337,36 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, if table not in mapper._pks_by_table: continue - pks = mapper._pks_by_table[table] - params = {} value_params = {} - - has_all_pks = True - has_all_defaults = True - has_version_id_generator = mapper.version_id_generator is not False \ - and mapper.version_id_col is not None - for col in mapper._cols_by_table[table]: - if has_version_id_generator and col is mapper.version_id_col: - val = mapper.version_id_generator(None) - params[col.key] = val + for col, propkey in mapper._col_to_propkey[table]: + if propkey in state_dict: + value = state_dict[propkey] + if isinstance(value, sql.ClauseElement): + value_params[col.key] = value + elif value is not None or ( + not col.primary_key and + not col.server_default and + not col.default): + params[col.key] = value else: - # pull straight from the dict for - # pending objects - prop = mapper._columntoproperty[col] - value = state_dict.get(prop.key, None) + if not col.server_default \ + and not col.default and not col.primary_key: + params[col.key] = None - if value is None: - if col in pks: - has_all_pks = False - elif col.default is None and \ - col.server_default is None: - params[col.key] = value - elif col.server_default is not None and \ - mapper.base_mapper.eager_defaults: - has_all_defaults = False + has_all_pks = mapper._pk_keys_by_table[table].issubset(params) - elif isinstance(value, sql.ClauseElement): - value_params[col] = value - else: - params[col.key] = value + if base_mapper.eager_defaults: + has_all_defaults = mapper._server_default_cols[table].\ + issubset(params) + else: + has_all_defaults = True + + if mapper.version_id_generator is not False \ + and mapper.version_id_col is not None and \ + mapper.version_id_col in mapper._cols_by_table[table]: + params[mapper.version_id_col.key] = \ + mapper.version_id_generator(None) insert.append((state, state_dict, params, mapper, connection, value_params, has_all_pks, @@ -572,30 +564,9 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, return delete -def _update_stmt_for_mapper(mapper, table, needs_version_id): - clause = sql.and_() - - for col in mapper._pks_by_table[table]: - clause.clauses.append(col == sql.bindparam(col._label, - type_=col.type)) - - if needs_version_id: - clause.clauses.append( - mapper.version_id_col == sql.bindparam( - mapper.version_id_col._label, - type_=mapper.version_id_col.type)) - - stmt = table.update(clause) - if mapper.base_mapper.eager_defaults: - stmt = stmt.return_defaults() - elif mapper.version_id_col is not None: - stmt = stmt.return_defaults(mapper.version_id_col) - - return stmt - - def _emit_update_statements(base_mapper, uowtransaction, - cached_connections, mapper, table, update): + cached_connections, mapper, table, update, + bookkeeping=True): """Emit UPDATE statements corresponding to value lists collected by _collect_update_commands().""" @@ -603,7 +574,25 @@ def _emit_update_statements(base_mapper, uowtransaction, table.c.contains_column(mapper.version_id_col) def update_stmt(): - return _update_stmt_for_mapper(mapper, table, needs_version_id) + clause = sql.and_() + + for col in mapper._pks_by_table[table]: + clause.clauses.append(col == sql.bindparam(col._label, + type_=col.type)) + + if needs_version_id: + clause.clauses.append( + mapper.version_id_col == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type)) + + stmt = table.update(clause) + if mapper.base_mapper.eager_defaults: + stmt = stmt.return_defaults() + elif mapper.version_id_col is not None: + stmt = stmt.return_defaults(mapper.version_id_col) + + return stmt statement = base_mapper._memo(('update', table), update_stmt) @@ -624,15 +613,16 @@ def _emit_update_statements(base_mapper, uowtransaction, c = connection.execute( statement.values(value_params), params) - _postfetch( - mapper, - uowtransaction, - table, - state, - state_dict, - c, - c.context.compiled_parameters[0], - value_params) + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params) rows += c.rowcount else: multiparams = [rec[2] for rec in records] @@ -640,17 +630,18 @@ def _emit_update_statements(base_mapper, uowtransaction, execute(statement, multiparams) rows += c.rowcount - for state, state_dict, params, mapper, \ - connection, value_params in records: - _postfetch( - mapper, - uowtransaction, - table, - state, - state_dict, - c, - c.context.compiled_parameters[0], - value_params) + if bookkeeping: + for state, state_dict, params, mapper, \ + connection, value_params in records: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params) if connection.dialect.supports_sane_rowcount: if rows != len(records): @@ -667,7 +658,8 @@ def _emit_update_statements(base_mapper, uowtransaction, def _emit_insert_statements(base_mapper, uowtransaction, - cached_connections, mapper, table, insert): + cached_connections, mapper, table, insert, + bookkeeping=True): """Emit INSERT statements corresponding to value lists collected by _collect_insert_commands().""" @@ -676,11 +668,11 @@ def _emit_insert_statements(base_mapper, uowtransaction, for (connection, pkeys, hasvalue, has_all_pks, has_all_defaults), \ records in groupby(insert, lambda rec: (rec[4], - list(rec[2].keys()), + tuple(sorted(rec[2].keys())), bool(rec[5]), rec[6], rec[7]) ): - if \ + if not bookkeeping or \ ( has_all_defaults or not base_mapper.eager_defaults @@ -693,19 +685,20 @@ def _emit_insert_statements(base_mapper, uowtransaction, c = cached_connections[connection].\ execute(statement, multiparams) - for (state, state_dict, params, mapper_rec, - conn, value_params, has_all_pks, has_all_defaults), \ - last_inserted_params in \ - zip(records, c.context.compiled_parameters): - _postfetch( - mapper_rec, - uowtransaction, - table, - state, - state_dict, - c, - last_inserted_params, - value_params) + if bookkeeping: + for (state, state_dict, params, mapper_rec, + conn, value_params, has_all_pks, has_all_defaults), \ + last_inserted_params in \ + zip(records, c.context.compiled_parameters): + _postfetch( + mapper_rec, + uowtransaction, + table, + state, + state_dict, + c, + last_inserted_params, + value_params) else: if not has_all_defaults and base_mapper.eager_defaults: -- cgit v1.2.1 From 84cca0e28660b5d35c35195aa57c89b094fa897d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 18:30:14 -0400 Subject: dev --- lib/sqlalchemy/orm/persistence.py | 47 +++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 8d3e90cf4..f9e7eda28 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -34,26 +34,18 @@ def bulk_insert(mapper, mappings, uowtransaction): "not supported in bulk_insert()") connection = uowtransaction.transaction.connection(base_mapper) - value_params = {} for table, sub_mapper in base_mapper._sorted_tables.items(): if not mapper.isa(sub_mapper): continue - has_version_generator = mapper.version_id_generator is not False and \ - mapper.version_id_col is not None - records = [] - for mapping in mappings: - params = dict( - (col.key, mapping[propkey]) - for col, propkey in mapper._col_to_propkey[table] - if propkey in mapping - ) - - if has_version_generator: - params[mapper.version_id_col.key] = \ - mapper.version_id_generator(None) - + for ( + state, state_dict, params, mapper, + connection, value_params, has_all_pks, + has_all_defaults) in _collect_insert_commands(table, ( + (None, mapping, sub_mapper, connection) + for mapping in mappings) + ): records.append( (None, None, params, sub_mapper, connection, value_params, True, True) @@ -82,13 +74,13 @@ def bulk_update(mapper, mappings, uowtransaction): if not mapper.isa(sub_mapper): continue - label_pks = mapper._pks_by_table[table] + label_pks = sub_mapper._pks_by_table[table] if mapper.version_id_col is not None: label_pks = label_pks.union([mapper.version_id_col]) to_translate = dict( (propkey, col._label if col in label_pks else col.key) - for col, propkey in mapper._col_to_propkey[table] + for propkey, col in sub_mapper._propkey_to_col[table].items() ) records = [] @@ -350,7 +342,7 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): yield state, dict_, mapper, bool(state.key), connection -def _collect_insert_commands(table, states_to_insert): +def _collect_insert_commands(table, states_to_insert, bulk=False): """Identify sets of values to use in INSERT statements for a list of states. @@ -374,17 +366,20 @@ def _collect_insert_commands(table, states_to_insert): else: params[col.key] = value - for colkey in mapper._insert_cols_as_none[table].\ - difference(params).difference(value_params): - params[colkey] = None + if not bulk: + for colkey in mapper._insert_cols_as_none[table].\ + difference(params).difference(value_params): + params[colkey] = None - has_all_pks = mapper._pk_keys_by_table[table].issubset(params) + has_all_pks = mapper._pk_keys_by_table[table].issubset(params) - if mapper.base_mapper.eager_defaults: - has_all_defaults = mapper._server_default_cols[table].\ - issubset(params) + if mapper.base_mapper.eager_defaults: + has_all_defaults = mapper._server_default_cols[table].\ + issubset(params) + else: + has_all_defaults = True else: - has_all_defaults = True + has_all_defaults = has_all_pks = True if mapper.version_id_generator is not False \ and mapper.version_id_col is not None and \ -- cgit v1.2.1 From a251001f24e819f1ebc525948437563f52a3a226 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 18:52:53 -0400 Subject: dev --- lib/sqlalchemy/orm/persistence.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index f9e7eda28..145a7783a 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -34,26 +34,25 @@ def bulk_insert(mapper, mappings, uowtransaction): "not supported in bulk_insert()") connection = uowtransaction.transaction.connection(base_mapper) - for table, sub_mapper in base_mapper._sorted_tables.items(): - if not mapper.isa(sub_mapper): + for table, super_mapper in base_mapper._sorted_tables.items(): + if not mapper.isa(super_mapper): continue - records = [] - for ( - state, state_dict, params, mapper, - connection, value_params, has_all_pks, - has_all_defaults) in _collect_insert_commands(table, ( - (None, mapping, sub_mapper, connection) + records = ( + (None, None, params, super_mapper, + connection, value_params, True, True) + for + state, state_dict, params, mp, + conn, value_params, has_all_pks, + has_all_defaults in _collect_insert_commands(table, ( + (None, mapping, super_mapper, connection) for mapping in mappings) - ): - records.append( - (None, None, params, sub_mapper, - connection, value_params, True, True) ) + ) _emit_insert_statements(base_mapper, uowtransaction, cached_connections, - mapper, table, records, + super_mapper, table, records, bookkeeping=False) @@ -70,17 +69,17 @@ def bulk_update(mapper, mappings, uowtransaction): connection = uowtransaction.transaction.connection(base_mapper) value_params = {} - for table, sub_mapper in base_mapper._sorted_tables.items(): - if not mapper.isa(sub_mapper): + for table, super_mapper in base_mapper._sorted_tables.items(): + if not mapper.isa(super_mapper): continue - label_pks = sub_mapper._pks_by_table[table] + label_pks = super_mapper._pks_by_table[table] if mapper.version_id_col is not None: label_pks = label_pks.union([mapper.version_id_col]) to_translate = dict( (propkey, col._label if col in label_pks else col.key) - for propkey, col in sub_mapper._propkey_to_col[table].items() + for propkey, col in super_mapper._propkey_to_col[table].items() ) records = [] @@ -97,12 +96,12 @@ def bulk_update(mapper, mappings, uowtransaction): params[mapper.version_id_col._label]) records.append( - (None, None, params, sub_mapper, connection, value_params) + (None, None, params, super_mapper, connection, value_params) ) _emit_update_statements(base_mapper, uowtransaction, cached_connections, - mapper, table, records, + super_mapper, table, records, bookkeeping=False) -- cgit v1.2.1 From 91959122e0a12943e5ff9399024c65ad4d7489e1 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 19 Aug 2014 14:24:56 -0400 Subject: - refinements --- lib/sqlalchemy/orm/events.py | 12 ----- lib/sqlalchemy/orm/mapper.py | 4 ++ lib/sqlalchemy/orm/persistence.py | 107 +++++++++++++++++++++++++------------- lib/sqlalchemy/orm/session.py | 29 ++++------- lib/sqlalchemy/orm/unitofwork.py | 6 --- 5 files changed, 86 insertions(+), 72 deletions(-) diff --git a/lib/sqlalchemy/orm/events.py b/lib/sqlalchemy/orm/events.py index 37ea3071b..aa99673ba 100644 --- a/lib/sqlalchemy/orm/events.py +++ b/lib/sqlalchemy/orm/events.py @@ -1453,18 +1453,6 @@ class SessionEvents(event.Events): """ - def before_bulk_insert(self, session, flush_context, mapper, mappings): - """""" - - def after_bulk_insert(self, session, flush_context, mapper, mappings): - """""" - - def before_bulk_update(self, session, flush_context, mapper, mappings): - """""" - - def after_bulk_update(self, session, flush_context, mapper, mappings): - """""" - def after_begin(self, session, transaction, connection): """Execute after a transaction is begun on a connection diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 89c092b58..b98fbda42 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -2366,6 +2366,10 @@ class Mapper(InspectionAttr): def _primary_key_props(self): return [self._columntoproperty[col] for col in self.primary_key] + @_memoized_configured_property + def _primary_key_propkeys(self): + return set([prop.key for prop in self._primary_key_props]) + def _get_state_attr_by_column( self, state, dict_, column, passive=attributes.PASSIVE_RETURN_NEVER_SET): diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 145a7783a..9c0008925 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -23,17 +23,22 @@ from ..sql import expression from . import loading -def bulk_insert(mapper, mappings, uowtransaction): +def _bulk_insert(mapper, mappings, session_transaction, isstates): base_mapper = mapper.base_mapper cached_connections = _cached_connection_dict(base_mapper) - if uowtransaction.session.connection_callable: + if session_transaction.session.connection_callable: raise NotImplementedError( "connection_callable / per-instance sharding " "not supported in bulk_insert()") - connection = uowtransaction.transaction.connection(base_mapper) + if isstates: + mappings = [state.dict for state in mappings] + else: + mappings = list(mappings) + + connection = session_transaction.connection(base_mapper) for table, super_mapper in base_mapper._sorted_tables.items(): if not mapper.isa(super_mapper): continue @@ -45,61 +50,55 @@ def bulk_insert(mapper, mappings, uowtransaction): state, state_dict, params, mp, conn, value_params, has_all_pks, has_all_defaults in _collect_insert_commands(table, ( - (None, mapping, super_mapper, connection) - for mapping in mappings) + (None, mapping, mapper, connection) + for mapping in mappings), + bulk=True ) ) - _emit_insert_statements(base_mapper, uowtransaction, + _emit_insert_statements(base_mapper, None, cached_connections, super_mapper, table, records, bookkeeping=False) -def bulk_update(mapper, mappings, uowtransaction): +def _bulk_update(mapper, mappings, session_transaction, isstates): base_mapper = mapper.base_mapper cached_connections = _cached_connection_dict(base_mapper) - if uowtransaction.session.connection_callable: + def _changed_dict(mapper, state): + return dict( + (k, v) + for k, v in state.dict.items() if k in state.committed_state or k + in mapper._primary_key_propkeys + ) + + if isstates: + mappings = [_changed_dict(mapper, state) for state in mappings] + else: + mappings = list(mappings) + + if session_transaction.session.connection_callable: raise NotImplementedError( "connection_callable / per-instance sharding " "not supported in bulk_update()") - connection = uowtransaction.transaction.connection(base_mapper) + connection = session_transaction.connection(base_mapper) value_params = {} + for table, super_mapper in base_mapper._sorted_tables.items(): if not mapper.isa(super_mapper): continue - label_pks = super_mapper._pks_by_table[table] - if mapper.version_id_col is not None: - label_pks = label_pks.union([mapper.version_id_col]) - - to_translate = dict( - (propkey, col._label if col in label_pks else col.key) - for propkey, col in super_mapper._propkey_to_col[table].items() + records = ( + (None, None, params, super_mapper, connection, value_params) + for + params in _collect_bulk_update_commands(mapper, table, mappings) ) - records = [] - for mapping in mappings: - params = dict( - (to_translate[k], v) for k, v in mapping.items() - ) - - if mapper.version_id_generator is not False and \ - mapper.version_id_col is not None and \ - mapper.version_id_col.key not in params: - params[mapper.version_id_col.key] = \ - mapper.version_id_generator( - params[mapper.version_id_col._label]) - - records.append( - (None, None, params, super_mapper, connection, value_params) - ) - - _emit_update_statements(base_mapper, uowtransaction, + _emit_update_statements(base_mapper, None, cached_connections, super_mapper, table, records, bookkeeping=False) @@ -360,7 +359,7 @@ def _collect_insert_commands(table, states_to_insert, bulk=False): col = propkey_to_col[propkey] if value is None: continue - elif isinstance(value, sql.ClauseElement): + elif not bulk and isinstance(value, sql.ClauseElement): value_params[col.key] = value else: params[col.key] = value @@ -481,6 +480,44 @@ def _collect_update_commands(uowtransaction, table, states_to_update): state, state_dict, params, mapper, connection, value_params) +def _collect_bulk_update_commands(mapper, table, mappings): + label_pks = mapper._pks_by_table[table] + if mapper.version_id_col is not None: + label_pks = label_pks.union([mapper.version_id_col]) + + to_translate = dict( + (propkey, col.key if col not in label_pks else col._label) + for propkey, col in mapper._propkey_to_col[table].items() + ) + + for mapping in mappings: + params = dict( + (to_translate[k], mapping[k]) for k in to_translate + if k in mapping and k not in mapper._primary_key_propkeys + ) + + if not params: + continue + + try: + params.update( + (to_translate[k], mapping[k]) for k in + mapper._primary_key_propkeys.intersection(to_translate) + ) + except KeyError as ke: + raise orm_exc.FlushError( + "Can't update table using NULL for primary " + "key attribute: %s" % ke) + + if mapper.version_id_generator is not False and \ + mapper.version_id_col is not None and \ + mapper.version_id_col.key not in params: + params[mapper.version_id_col.key] = \ + mapper.version_id_generator( + params[mapper.version_id_col._label]) + + yield params + def _collect_post_update_commands(base_mapper, uowtransaction, table, states_to_update, post_update_cols): diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 3199a4332..968868e84 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -21,6 +21,7 @@ from .base import ( _none_set, state_str, instance_str ) import itertools +from . import persistence from .unitofwork import UOWTransaction from . import state as statelib import sys @@ -2040,37 +2041,27 @@ class Session(_SessionClassMethods): (attributes.instance_state(obj) for obj in objects), lambda state: (state.mapper, state.key is not None) ): - if isupdate: - self.bulk_update_mappings(mapper, (s.dict for s in states)) - else: - self.bulk_insert_mappings(mapper, (s.dict for s in states)) + self._bulk_save_mappings(mapper, states, isupdate, True) def bulk_insert_mappings(self, mapper, mappings): - self._bulk_save_mappings(mapper, mappings, False) + self._bulk_save_mappings(mapper, mappings, False, False) def bulk_update_mappings(self, mapper, mappings): - self._bulk_save_mappings(mapper, mappings, True) + self._bulk_save_mappings(mapper, mappings, True, False) - def _bulk_save_mappings(self, mapper, mappings, isupdate): + def _bulk_save_mappings(self, mapper, mappings, isupdate, isstates): mapper = _class_to_mapper(mapper) self._flushing = True - flush_context = UOWTransaction(self) - flush_context.transaction = transaction = self.begin( + transaction = self.begin( subtransactions=True) try: if isupdate: - self.dispatch.before_bulk_update( - self, flush_context, mapper, mappings) - flush_context.bulk_update(mapper, mappings) - self.dispatch.after_bulk_update( - self, flush_context, mapper, mappings) + persistence._bulk_update( + mapper, mappings, transaction, isstates) else: - self.dispatch.before_bulk_insert( - self, flush_context, mapper, mappings) - flush_context.bulk_insert(mapper, mappings) - self.dispatch.after_bulk_insert( - self, flush_context, mapper, mappings) + persistence._bulk_insert( + mapper, mappings, transaction, isstates) transaction.commit() except: diff --git a/lib/sqlalchemy/orm/unitofwork.py b/lib/sqlalchemy/orm/unitofwork.py index b3a1519c5..05265b13f 100644 --- a/lib/sqlalchemy/orm/unitofwork.py +++ b/lib/sqlalchemy/orm/unitofwork.py @@ -394,12 +394,6 @@ class UOWTransaction(object): if other: self.session._register_newly_persistent(other) - def bulk_insert(self, mapper, mappings): - persistence.bulk_insert(mapper, mappings, self) - - def bulk_update(self, mapper, mappings): - persistence.bulk_update(mapper, mappings, self) - class IterateMappersMixin(object): def _mappers(self, uow): -- cgit v1.2.1 From fcea5c86d3a9097caa04e2e35fa6404a3ef32044 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 19 Aug 2014 18:26:11 -0400 Subject: - rename mapper._primary_key_props to mapper._identity_key_props - ensure bulk update is using all PK cols for all tables --- lib/sqlalchemy/orm/mapper.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 63d23e31d..31c17e69e 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1244,7 +1244,7 @@ class Mapper(InspectionAttr): self._readonly_props = set( self._columntoproperty[col] for col in self._columntoproperty - if self._columntoproperty[col] not in self._primary_key_props and + if self._columntoproperty[col] not in self._identity_key_props and (not hasattr(col, 'table') or col.table not in self._cols_by_table)) @@ -2359,19 +2359,23 @@ class Mapper(InspectionAttr): manager[prop.key]. impl.get(state, dict_, attributes.PASSIVE_RETURN_NEVER_SET) - for prop in self._primary_key_props + for prop in self._identity_key_props ] @_memoized_configured_property - def _primary_key_props(self): - # TODO: this should really be called "identity key props", - # as it does not necessarily include primary key columns within - # individual tables + def _identity_key_props(self): return [self._columntoproperty[col] for col in self.primary_key] + @_memoized_configured_property + def _all_pk_props(self): + collection = set() + for table in self.tables: + collection.update(self._pks_by_table[table]) + return collection + @_memoized_configured_property def _primary_key_propkeys(self): - return set([prop.key for prop in self._primary_key_props]) + return set([prop.key for prop in self._all_pk_props]) def _get_state_attr_by_column( self, state, dict_, column, -- cgit v1.2.1 From d006e9cc2a84a05b46c480ad0c4b429036470e79 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 14:59:16 -0400 Subject: - skip these methods --- test/orm/test_session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/orm/test_session.py b/test/orm/test_session.py index 186b7a781..b2c8b5f02 100644 --- a/test/orm/test_session.py +++ b/test/orm/test_session.py @@ -1585,7 +1585,9 @@ class SessionInterface(fixtures.TestBase): raises_('refresh', user_arg) instance_methods = self._public_session_methods() \ - - self._class_methods + - self._class_methods - set([ + 'bulk_update_mappings', 'bulk_insert_mappings', + 'bulk_save_objects']) eq_(watchdog, instance_methods, watchdog.symmetric_difference(instance_methods)) -- cgit v1.2.1 From cb8f5c010b396dd83bdc1e4408787383f3c41d05 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 16:16:47 -0400 Subject: - test for postfetch->sync.populate() having importance during an UPDATE at the per-table level --- test/orm/test_naturalpks.py | 74 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/test/orm/test_naturalpks.py b/test/orm/test_naturalpks.py index a4e982f84..709e1c0b1 100644 --- a/test/orm/test_naturalpks.py +++ b/test/orm/test_naturalpks.py @@ -1205,3 +1205,77 @@ class JoinedInheritanceTest(fixtures.MappedTest): eq_(e1.boss_name, 'pointy haired') eq_(e2.boss_name, 'pointy haired') + + +class JoinedInheritancePKOnFKTest(fixtures.MappedTest): + """Test cascades of pk->non-pk/fk on joined table inh.""" + + # mssql doesn't allow ON UPDATE on self-referential keys + __unsupported_on__ = ('mssql',) + + __requires__ = 'skip_mysql_on_windows', + __backend__ = True + + @classmethod + def define_tables(cls, metadata): + fk_args = _backend_specific_fk_args() + + Table( + 'person', metadata, + Column('name', String(50), primary_key=True), + Column('type', String(50), nullable=False), + test_needs_fk=True) + + Table( + 'engineer', metadata, + Column('id', Integer, primary_key=True), + Column( + 'person_name', String(50), + ForeignKey('person.name', **fk_args)), + Column('primary_language', String(50)), + test_needs_fk=True + ) + + @classmethod + def setup_classes(cls): + + class Person(cls.Comparable): + pass + + class Engineer(Person): + pass + + def _test_pk(self, passive_updates): + Person, person, Engineer, engineer = ( + self.classes.Person, self.tables.person, + self.classes.Engineer, self.tables.engineer) + + mapper( + Person, person, polymorphic_on=person.c.type, + polymorphic_identity='person', passive_updates=passive_updates) + mapper( + Engineer, engineer, inherits=Person, + polymorphic_identity='engineer') + + sess = sa.orm.sessionmaker()() + + e1 = Engineer(name='dilbert', primary_language='java') + sess.add(e1) + sess.commit() + e1.name = 'wally' + e1.primary_language = 'c++' + + sess.flush() + + eq_(e1.person_name, 'wally') + + sess.expire_all() + eq_(e1.primary_language, "c++") + + @testing.requires.on_update_cascade + def test_pk_passive(self): + self._test_pk(True) + + #@testing.requires.non_updating_cascade + def test_pk_nonpassive(self): + self._test_pk(False) -- cgit v1.2.1 From db70b6e79e263c137f4d282c9c600417636afa25 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 17:15:20 -0400 Subject: - that's it, feature is finished, needs tests --- lib/sqlalchemy/orm/persistence.py | 195 +++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 106 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index c2750eeb3..aa10da9f4 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -15,7 +15,7 @@ in unitofwork.py. """ import operator -from itertools import groupby +from itertools import groupby, chain from .. import sql, util, exc as sa_exc, schema from . import attributes, sync, exc as orm_exc, evaluator from .base import state_str, _attr_as_key @@ -86,17 +86,16 @@ def _bulk_update(mapper, mappings, session_transaction, isstates): connection = session_transaction.connection(base_mapper) - value_params = {} - for table, super_mapper in base_mapper._sorted_tables.items(): if not mapper.isa(super_mapper): continue - records = ( - (None, None, params, super_mapper, connection, value_params) - for - params in _collect_bulk_update_commands(mapper, table, mappings) - ) + records = _collect_update_commands(None, table, ( + (None, mapping, mapper, connection, + (mapping[mapper._version_id_prop.key] + if mapper._version_id_prop else None)) + for mapping in mappings + ), bulk=True) _emit_update_statements(base_mapper, None, cached_connections, @@ -158,17 +157,16 @@ def save_obj( _finalize_insert_update_commands( base_mapper, uowtransaction, - ( - (state, state_dict, mapper, connection, False) - for state, state_dict, mapper, connection in states_to_insert - ) - ) - _finalize_insert_update_commands( - base_mapper, uowtransaction, - ( - (state, state_dict, mapper, connection, True) - for state, state_dict, mapper, connection, - update_version_id in states_to_update + chain( + ( + (state, state_dict, mapper, connection, False) + for state, state_dict, mapper, connection in states_to_insert + ), + ( + (state, state_dict, mapper, connection, True) + for state, state_dict, mapper, connection, + update_version_id in states_to_update + ) ) ) @@ -394,7 +392,9 @@ def _collect_insert_commands(table, states_to_insert, bulk=False): has_all_defaults) -def _collect_update_commands(uowtransaction, table, states_to_update): +def _collect_update_commands( + uowtransaction, table, states_to_update, + bulk=False): """Identify sets of values to use in UPDATE statements for a list of states. @@ -414,23 +414,32 @@ def _collect_update_commands(uowtransaction, table, states_to_update): pks = mapper._pks_by_table[table] - params = {} value_params = {} propkey_to_col = mapper._propkey_to_col[table] - for propkey in set(propkey_to_col).intersection(state.committed_state): - value = state_dict[propkey] - col = propkey_to_col[propkey] - - if not state.manager[propkey].impl.is_equal( - value, state.committed_state[propkey]): - if isinstance(value, sql.ClauseElement): - value_params[col] = value - else: - params[col.key] = value + if bulk: + params = dict( + (propkey_to_col[propkey].key, state_dict[propkey]) + for propkey in + set(propkey_to_col).intersection(state_dict) + ) + else: + params = {} + for propkey in set(propkey_to_col).intersection( + state.committed_state): + value = state_dict[propkey] + col = propkey_to_col[propkey] + + if not state.manager[propkey].impl.is_equal( + value, state.committed_state[propkey]): + if isinstance(value, sql.ClauseElement): + value_params[col] = value + else: + params[col.key] = value - if update_version_id is not None: + if update_version_id is not None and \ + mapper.version_id_col in mapper._cols_by_table[table]: col = mapper.version_id_col params[col._label] = update_version_id @@ -442,24 +451,33 @@ def _collect_update_commands(uowtransaction, table, states_to_update): if not (params or value_params): continue - pk_params = {} - for col in pks: - propkey = mapper._columntoproperty[col].key - history = state.manager[propkey].impl.get_history( - state, state_dict, attributes.PASSIVE_OFF) - - if history.added: - if not history.deleted or \ - ("pk_cascaded", state, col) in \ - uowtransaction.attributes: - pk_params[col._label] = history.added[0] - params.pop(col.key, None) + if bulk: + pk_params = dict( + (propkey_to_col[propkey]._label, state_dict.get(propkey)) + for propkey in + set(propkey_to_col). + intersection(mapper._pk_keys_by_table[table]) + ) + else: + pk_params = {} + for col in pks: + propkey = mapper._columntoproperty[col].key + + history = state.manager[propkey].impl.get_history( + state, state_dict, attributes.PASSIVE_OFF) + + if history.added: + if not history.deleted or \ + ("pk_cascaded", state, col) in \ + uowtransaction.attributes: + pk_params[col._label] = history.added[0] + params.pop(col.key, None) + else: + # else, use the old value to locate the row + pk_params[col._label] = history.deleted[0] + params[col.key] = history.added[0] else: - # else, use the old value to locate the row - pk_params[col._label] = history.deleted[0] - params[col.key] = history.added[0] - else: - pk_params[col._label] = history.unchanged[0] + pk_params[col._label] = history.unchanged[0] if params or value_params: if None in pk_params.values(): @@ -471,44 +489,6 @@ def _collect_update_commands(uowtransaction, table, states_to_update): state, state_dict, params, mapper, connection, value_params) -def _collect_bulk_update_commands(mapper, table, mappings): - label_pks = mapper._pks_by_table[table] - if mapper.version_id_col is not None: - label_pks = label_pks.union([mapper.version_id_col]) - - to_translate = dict( - (propkey, col.key if col not in label_pks else col._label) - for propkey, col in mapper._propkey_to_col[table].items() - ) - - for mapping in mappings: - params = dict( - (to_translate[k], mapping[k]) for k in to_translate - if k in mapping and k not in mapper._primary_key_propkeys - ) - - if not params: - continue - - try: - params.update( - (to_translate[k], mapping[k]) for k in - mapper._primary_key_propkeys.intersection(to_translate) - ) - except KeyError as ke: - raise orm_exc.FlushError( - "Can't update table using NULL for primary " - "key attribute: %s" % ke) - - if mapper.version_id_generator is not False and \ - mapper.version_id_col is not None and \ - mapper.version_id_col.key not in params: - params[mapper.version_id_col.key] = \ - mapper.version_id_generator( - params[mapper.version_id_col._label]) - - yield params - def _collect_post_update_commands(base_mapper, uowtransaction, table, states_to_update, post_update_cols): @@ -569,7 +549,7 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, "key value") if update_version_id is not None and \ - table.c.contains_column(mapper.version_id_col): + mapper.version_id_col in mapper._cols_by_table[table]: params[mapper.version_id_col.key] = update_version_id yield params, connection @@ -581,7 +561,7 @@ def _emit_update_statements(base_mapper, uowtransaction, by _collect_update_commands().""" needs_version_id = mapper.version_id_col is not None and \ - table.c.contains_column(mapper.version_id_col) + mapper.version_id_col in mapper._cols_by_table[table] def update_stmt(): clause = sql.and_() @@ -610,9 +590,9 @@ def _emit_update_statements(base_mapper, uowtransaction, records in groupby( update, lambda rec: ( - rec[4], - tuple(sorted(rec[2])), - bool(rec[5]))): + rec[4], # connection + set(rec[2]), # set of parameter keys + bool(rec[5]))): # whether or not we have "value" parameters rows = 0 records = list(records) @@ -692,12 +672,14 @@ def _emit_insert_statements(base_mapper, uowtransaction, statement = base_mapper._memo(('insert', table), table.insert) for (connection, pkeys, hasvalue, has_all_pks, has_all_defaults), \ - records in groupby(insert, - lambda rec: (rec[4], - tuple(sorted(rec[2].keys())), - bool(rec[5]), - rec[6], rec[7]) - ): + records in groupby( + insert, + lambda rec: ( + rec[4], # connection + set(rec[2]), # parameter keys + bool(rec[5]), # whether we have "value" parameters + rec[6], + rec[7])): if not bookkeeping or \ ( has_all_defaults @@ -785,7 +767,10 @@ def _emit_post_update_statements(base_mapper, uowtransaction, # also group them into common (connection, cols) sets # to support executemany(). for key, grouper in groupby( - update, lambda rec: (rec[1], sorted(rec[0])) + update, lambda rec: ( + rec[1], # connection + set(rec[0]) # parameter keys + ) ): connection = key[0] multiparams = [params for params, conn in grouper] @@ -799,7 +784,7 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, by _collect_delete_commands().""" need_version_id = mapper.version_id_col is not None and \ - table.c.contains_column(mapper.version_id_col) + mapper.version_id_col in mapper._cols_by_table[table] def delete_stmt(): clause = sql.and_() @@ -821,12 +806,9 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, statement = base_mapper._memo(('delete', table), delete_stmt) for connection, recs in groupby( delete, - lambda rec: rec[1] + lambda rec: rec[1] # connection ): - del_objects = [ - params - for params, connection in recs - ] + del_objects = [params for params, connection in recs] connection = cached_connections[connection] @@ -931,7 +913,8 @@ def _postfetch(mapper, uowtransaction, table, postfetch_cols = result.context.postfetch_cols returning_cols = result.context.returning_cols - if mapper.version_id_col is not None: + if mapper.version_id_col is not None and \ + mapper.version_id_col in mapper._cols_by_table[table]: prefetch_cols = list(prefetch_cols) + [mapper.version_id_col] if returning_cols: -- cgit v1.2.1 From ccfd26d96916cc7953f1fefa8abed53d4f696c4c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 2 Sep 2014 19:23:09 -0400 Subject: - add options to get back pk defaults for inserts. times spent start getting barely different... --- lib/sqlalchemy/orm/persistence.py | 37 ++++++++++++++++++++++++++----------- lib/sqlalchemy/orm/session.py | 16 +++++++++------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 198eeb46f..2a697a6f9 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -23,7 +23,8 @@ from ..sql import expression from . import loading -def _bulk_insert(mapper, mappings, session_transaction, isstates): +def _bulk_insert( + mapper, mappings, session_transaction, isstates, return_defaults): base_mapper = mapper.base_mapper cached_connections = _cached_connection_dict(base_mapper) @@ -34,7 +35,11 @@ def _bulk_insert(mapper, mappings, session_transaction, isstates): "not supported in bulk_insert()") if isstates: - mappings = [state.dict for state in mappings] + if return_defaults: + states = [(state, state.dict) for state in mappings] + mappings = [dict_ for (state, dict_) in states] + else: + mappings = [state.dict for state in mappings] else: mappings = list(mappings) @@ -44,22 +49,30 @@ def _bulk_insert(mapper, mappings, session_transaction, isstates): continue records = ( - (None, None, params, super_mapper, - connection, value_params, True, True) + (None, state_dict, params, super_mapper, + connection, value_params, has_all_pks, has_all_defaults) for state, state_dict, params, mp, conn, value_params, has_all_pks, has_all_defaults in _collect_insert_commands(table, ( (None, mapping, mapper, connection) for mapping in mappings), - bulk=True + bulk=True, return_defaults=return_defaults ) ) - _emit_insert_statements(base_mapper, None, cached_connections, super_mapper, table, records, - bookkeeping=False) + bookkeeping=return_defaults) + + if return_defaults and isstates: + identity_cls = mapper._identity_class + identity_props = [p.key for p in mapper._identity_key_props] + for state, dict_ in states: + state.key = ( + identity_cls, + tuple([dict_[key] for key in identity_props]) + ) def _bulk_update(mapper, mappings, session_transaction, isstates): @@ -341,7 +354,9 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): state, dict_, mapper, connection, update_version_id) -def _collect_insert_commands(table, states_to_insert, bulk=False): +def _collect_insert_commands( + table, states_to_insert, + bulk=False, return_defaults=False): """Identify sets of values to use in INSERT statements for a list of states. @@ -370,6 +385,7 @@ def _collect_insert_commands(table, states_to_insert, bulk=False): difference(params).difference(value_params): params[colkey] = None + if not bulk or return_defaults: has_all_pks = mapper._pk_keys_by_table[table].issubset(params) if mapper.base_mapper.eager_defaults: @@ -884,9 +900,8 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, states): toload_now.extend(state._unloaded_non_object) elif mapper.version_id_col is not None and \ mapper.version_id_generator is False: - prop = mapper._columntoproperty[mapper.version_id_col] - if prop.key in state.unloaded: - toload_now.extend([prop.key]) + if mapper._version_id_prop.key in state.unloaded: + toload_now.extend([mapper._version_id_prop.key]) if toload_now: state.key = base_mapper._identity_key_from_state(state) diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index e075b9c71..1611688b0 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -2036,20 +2036,22 @@ class Session(_SessionClassMethods): with util.safe_reraise(): transaction.rollback(_capture_exception=True) - def bulk_save_objects(self, objects): + def bulk_save_objects(self, objects, return_defaults=False): for (mapper, isupdate), states in itertools.groupby( (attributes.instance_state(obj) for obj in objects), lambda state: (state.mapper, state.key is not None) ): - self._bulk_save_mappings(mapper, states, isupdate, True) + self._bulk_save_mappings( + mapper, states, isupdate, True, return_defaults) - def bulk_insert_mappings(self, mapper, mappings): - self._bulk_save_mappings(mapper, mappings, False, False) + def bulk_insert_mappings(self, mapper, mappings, return_defaults=False): + self._bulk_save_mappings(mapper, mappings, False, False, return_defaults) def bulk_update_mappings(self, mapper, mappings): - self._bulk_save_mappings(mapper, mappings, True, False) + self._bulk_save_mappings(mapper, mappings, True, False, False) - def _bulk_save_mappings(self, mapper, mappings, isupdate, isstates): + def _bulk_save_mappings( + self, mapper, mappings, isupdate, isstates, return_defaults): mapper = _class_to_mapper(mapper) self._flushing = True @@ -2061,7 +2063,7 @@ class Session(_SessionClassMethods): mapper, mappings, transaction, isstates) else: persistence._bulk_insert( - mapper, mappings, transaction, isstates) + mapper, mappings, transaction, isstates, return_defaults) transaction.commit() except: -- cgit v1.2.1 From 9494ca00d4451448fd4473c03dff8459051224a2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 2 Sep 2014 19:46:55 -0400 Subject: - lets start exampling this stuff --- examples/performance/__init__.py | 0 examples/performance/inserts.py | 148 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 examples/performance/__init__.py create mode 100644 examples/performance/inserts.py diff --git a/examples/performance/__init__.py b/examples/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/performance/inserts.py b/examples/performance/inserts.py new file mode 100644 index 000000000..469501d8d --- /dev/null +++ b/examples/performance/inserts.py @@ -0,0 +1,148 @@ +import time + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import Session + +Base = declarative_base() +engine = None + + +class Customer(Base): + __tablename__ = "customer" + id = Column(Integer, primary_key=True) + name = Column(String(255)) + description = Column(String(255)) + + +def setup_database(): + global engine + engine = create_engine("sqlite:///insert_speed.db", echo=False) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + +_tests = [] + + +def _test(fn): + _tests.append(fn) + return fn + + +@_test +def test_flush_no_pk(n): + """Individual INSERT statements via the ORM, calling upon last row id""" + session = Session(bind=engine) + for chunk in range(0, n, 1000): + session.add_all([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i) + for i in range(chunk, chunk + 1000) + ]) + session.flush() + session.commit() + + +@_test +def test_bulk_save_return_pks(n): + """Individual INSERT statements in "bulk", but calling upon last row id""" + session = Session(bind=engine) + session.bulk_save_objects([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ], return_defaults=True) + session.commit() + + +@_test +def test_flush_pk_given(n): + """Batched INSERT statements via the ORM, PKs already defined""" + session = Session(bind=engine) + for chunk in range(0, n, 1000): + session.add_all([ + Customer( + id=i + 1, + name='customer name %d' % i, + description='customer description %d' % i) + for i in range(chunk, chunk + 1000) + ]) + session.flush() + session.commit() + + +@_test +def test_bulk_save(n): + """Batched INSERT statements via the ORM in "bulk", discarding PK values.""" + session = Session(bind=engine) + session.bulk_save_objects([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ]) + session.commit() + + +@_test +def test_bulk_insert_mappings(n): + """Batched INSERT statements via the ORM "bulk", using dictionaries instead of objects""" + session = Session(bind=engine) + session.bulk_insert_mappings(Customer, [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ]) + session.commit() + + +@_test +def test_core_insert(n): + """A single Core INSERT construct inserting mappings in bulk.""" + conn = engine.connect() + conn.execute( + Customer.__table__.insert(), + [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ]) + + +@_test +def test_sqlite_raw(n): + """pysqlite's pure C API inserting rows in bulk, no pure Python at all""" + conn = engine.raw_connection() + cursor = conn.cursor() + cursor.executemany( + "INSERT INTO customer (name, description) VALUES(:name, :description)", + [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ] + ) + conn.commit() + + +def run_tests(n): + for fn in _tests: + setup_database() + now = time.time() + fn(n) + total = time.time() - now + + print("Test: %s; Total time %s" % (fn.__doc__, total)) + +if __name__ == '__main__': + run_tests(100000) -- cgit v1.2.1 From 07d061a17b3fbad89df97e57350b4d0c132408c2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 3 Sep 2014 14:49:26 -0400 Subject: - wip --- examples/performance/__init__.py | 183 +++++++++++++++++++++++++++++++++++ examples/performance/bulk_inserts.py | 132 +++++++++++++++++++++++++ examples/performance/inserts.py | 148 ---------------------------- 3 files changed, 315 insertions(+), 148 deletions(-) create mode 100644 examples/performance/bulk_inserts.py delete mode 100644 examples/performance/inserts.py diff --git a/examples/performance/__init__.py b/examples/performance/__init__.py index e69de29bb..ae914db96 100644 --- a/examples/performance/__init__.py +++ b/examples/performance/__init__.py @@ -0,0 +1,183 @@ +"""A performance profiling suite for a variety of SQLAlchemy use cases. + +The suites here each focus on some specific type of use case, one which +has a particular performance profile: + +* bulk inserts +* individual inserts, with or without transactions +* fetching large numbers of rows +* running lots of small queries + +All suites include a variety of use patterns with both the Core and +ORM, and are sorted in order of performance from worst to greatest, +inversely based on amount of functionality provided by SQLAlchemy, +greatest to least (these two things generally correspond perfectly). + +Each suite is run as a module, and provides a consistent command line +interface:: + + $ python -m examples.performance.bulk_inserts --profile --num 1000 + +Using ``--help`` will allow all options:: + + $ python -m examples.performance.bulk_inserts --help +usage: bulk_inserts.py [-h] [--test TEST] [--dburl DBURL] [--num NUM] + [--profile] [--dump] [--runsnake] [--echo] + +optional arguments: + -h, --help show this help message and exit + --test TEST run specific test name + --dburl DBURL database URL, default sqlite:///profile.db + --num NUM Number of iterations/items/etc for tests, default 100000 + --profile run profiling and dump call counts + --dump dump full call profile (implies --profile) + --runsnake invoke runsnakerun (implies --profile) + --echo Echo SQL output + + +""" +import argparse +import cProfile +import StringIO +import pstats +import os +import time + + + +class Profiler(object): + tests = [] + + def __init__(self, setup, options): + self.setup = setup + self.test = options.test + self.dburl = options.dburl + self.runsnake = options.runsnake + self.profile = options.profile + self.dump = options.dump + self.num = options.num + self.echo = options.echo + self.stats = [] + + @classmethod + def profile(cls, fn): + cls.tests.append(fn) + return fn + + def run(self): + if self.test: + tests = [fn for fn in self.tests if fn.__name__ == self.test] + if not tests: + raise ValueError("No such test: %s" % self.test) + else: + tests = self.tests + + print("Tests to run: %s" % ", ".join([t.__name__ for t in tests])) + for test in tests: + self._run_test(test) + self.stats[-1].report() + + def _run_with_profile(self, fn): + pr = cProfile.Profile() + pr.enable() + try: + result = fn(self.num) + finally: + pr.disable() + + output = StringIO.StringIO() + stats = pstats.Stats(pr, stream=output).sort_stats('cumulative') + + self.stats.append(TestResult(self, fn, stats=stats)) + return result + + def _run_with_time(self, fn): + now = time.time() + try: + return fn(self.num) + finally: + total = time.time() - now + self.stats.append(TestResult(self, fn, total_time=total)) + + def _run_test(self, fn): + self.setup(self.dburl, self.echo) + if self.profile or self.runsnake or self.dump: + self._run_with_profile(fn) + else: + self._run_with_time(fn) + + @classmethod + def main(cls, setup): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--test", type=str, + help="run specific test name" + ) + parser.add_argument( + '--dburl', type=str, default="sqlite:///profile.db", + help="database URL, default sqlite:///profile.db" + ) + parser.add_argument( + '--num', type=int, default=100000, + help="Number of iterations/items/etc for tests, default 100000" + ) + parser.add_argument( + '--profile', action='store_true', + help='run profiling and dump call counts') + parser.add_argument( + '--dump', action='store_true', + help='dump full call profile (implies --profile)') + parser.add_argument( + '--runsnake', action='store_true', + help='invoke runsnakerun (implies --profile)') + parser.add_argument( + '--echo', action='store_true', + help="Echo SQL output" + ) + args = parser.parse_args() + + args.profile = args.profile or args.dump or args.runsnake + + Profiler(setup, args).run() + + +class TestResult(object): + def __init__(self, profile, test, stats=None, total_time=None): + self.profile = profile + self.test = test + self.stats = stats + self.total_time = total_time + + def report(self): + print(self._summary()) + if self.profile.profile: + self.report_stats() + + def _summary(self): + summary = "%s : %s (%d iterations)" % ( + self.test.__name__, self.test.__doc__, self.profile.num) + if self.total_time: + summary += "; total time %f sec" % self.total_time + if self.stats: + summary += "; total fn calls %d" % self.stats.total_calls + return summary + + def report_stats(self): + if self.profile.runsnake: + self._runsnake() + elif self.profile.dump: + self._dump() + + def _dump(self): + self.stats.sort_stats('time', 'calls') + self.stats.print_stats() + + def _runsnake(self): + filename = "%s.profile" % self.test.__name__ + try: + self.stats.dump_stats(filename) + os.system("runsnake %s" % filename) + finally: + os.remove(filename) + diff --git a/examples/performance/bulk_inserts.py b/examples/performance/bulk_inserts.py new file mode 100644 index 000000000..42ab920a6 --- /dev/null +++ b/examples/performance/bulk_inserts.py @@ -0,0 +1,132 @@ +from . import Profiler + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import Session + +Base = declarative_base() +engine = None + + +class Customer(Base): + __tablename__ = "customer" + id = Column(Integer, primary_key=True) + name = Column(String(255)) + description = Column(String(255)) + + +def setup_database(dburl, echo): + global engine + engine = create_engine(dburl, echo=echo) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + +@Profiler.profile +def test_flush_no_pk(n): + """Individual INSERT statements via the ORM, calling upon last row id""" + session = Session(bind=engine) + for chunk in range(0, n, 1000): + session.add_all([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i) + for i in range(chunk, chunk + 1000) + ]) + session.flush() + session.commit() + + +@Profiler.profile +def test_bulk_save_return_pks(n): + """Individual INSERT statements in "bulk", but calling upon last row id""" + session = Session(bind=engine) + session.bulk_save_objects([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ], return_defaults=True) + session.commit() + + +@Profiler.profile +def test_flush_pk_given(n): + """Batched INSERT statements via the ORM, PKs already defined""" + session = Session(bind=engine) + for chunk in range(0, n, 1000): + session.add_all([ + Customer( + id=i + 1, + name='customer name %d' % i, + description='customer description %d' % i) + for i in range(chunk, chunk + 1000) + ]) + session.flush() + session.commit() + + +@Profiler.profile +def test_bulk_save(n): + """Batched INSERT statements via the ORM in "bulk", discarding PK values.""" + session = Session(bind=engine) + session.bulk_save_objects([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ]) + session.commit() + + +@Profiler.profile +def test_bulk_insert_mappings(n): + """Batched INSERT statements via the ORM "bulk", using dictionaries instead of objects""" + session = Session(bind=engine) + session.bulk_insert_mappings(Customer, [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ]) + session.commit() + + +@Profiler.profile +def test_core_insert(n): + """A single Core INSERT construct inserting mappings in bulk.""" + conn = engine.connect() + conn.execute( + Customer.__table__.insert(), + [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ]) + + +@Profiler.profile +def test_sqlite_raw(n): + """pysqlite's pure C API inserting rows in bulk, no pure Python at all""" + conn = engine.raw_connection() + cursor = conn.cursor() + cursor.executemany( + "INSERT INTO customer (name, description) VALUES(:name, :description)", + [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ] + ) + conn.commit() + + +if __name__ == '__main__': + Profiler.main(setup=setup_database) diff --git a/examples/performance/inserts.py b/examples/performance/inserts.py deleted file mode 100644 index 469501d8d..000000000 --- a/examples/performance/inserts.py +++ /dev/null @@ -1,148 +0,0 @@ -import time - -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, Integer, String, create_engine -from sqlalchemy.orm import Session - -Base = declarative_base() -engine = None - - -class Customer(Base): - __tablename__ = "customer" - id = Column(Integer, primary_key=True) - name = Column(String(255)) - description = Column(String(255)) - - -def setup_database(): - global engine - engine = create_engine("sqlite:///insert_speed.db", echo=False) - Base.metadata.drop_all(engine) - Base.metadata.create_all(engine) - -_tests = [] - - -def _test(fn): - _tests.append(fn) - return fn - - -@_test -def test_flush_no_pk(n): - """Individual INSERT statements via the ORM, calling upon last row id""" - session = Session(bind=engine) - for chunk in range(0, n, 1000): - session.add_all([ - Customer( - name='customer name %d' % i, - description='customer description %d' % i) - for i in range(chunk, chunk + 1000) - ]) - session.flush() - session.commit() - - -@_test -def test_bulk_save_return_pks(n): - """Individual INSERT statements in "bulk", but calling upon last row id""" - session = Session(bind=engine) - session.bulk_save_objects([ - Customer( - name='customer name %d' % i, - description='customer description %d' % i - ) - for i in range(n) - ], return_defaults=True) - session.commit() - - -@_test -def test_flush_pk_given(n): - """Batched INSERT statements via the ORM, PKs already defined""" - session = Session(bind=engine) - for chunk in range(0, n, 1000): - session.add_all([ - Customer( - id=i + 1, - name='customer name %d' % i, - description='customer description %d' % i) - for i in range(chunk, chunk + 1000) - ]) - session.flush() - session.commit() - - -@_test -def test_bulk_save(n): - """Batched INSERT statements via the ORM in "bulk", discarding PK values.""" - session = Session(bind=engine) - session.bulk_save_objects([ - Customer( - name='customer name %d' % i, - description='customer description %d' % i - ) - for i in range(n) - ]) - session.commit() - - -@_test -def test_bulk_insert_mappings(n): - """Batched INSERT statements via the ORM "bulk", using dictionaries instead of objects""" - session = Session(bind=engine) - session.bulk_insert_mappings(Customer, [ - dict( - name='customer name %d' % i, - description='customer description %d' % i - ) - for i in range(n) - ]) - session.commit() - - -@_test -def test_core_insert(n): - """A single Core INSERT construct inserting mappings in bulk.""" - conn = engine.connect() - conn.execute( - Customer.__table__.insert(), - [ - dict( - name='customer name %d' % i, - description='customer description %d' % i - ) - for i in range(n) - ]) - - -@_test -def test_sqlite_raw(n): - """pysqlite's pure C API inserting rows in bulk, no pure Python at all""" - conn = engine.raw_connection() - cursor = conn.cursor() - cursor.executemany( - "INSERT INTO customer (name, description) VALUES(:name, :description)", - [ - dict( - name='customer name %d' % i, - description='customer description %d' % i - ) - for i in range(n) - ] - ) - conn.commit() - - -def run_tests(n): - for fn in _tests: - setup_database() - now = time.time() - fn(n) - total = time.time() - now - - print("Test: %s; Total time %s" % (fn.__doc__, total)) - -if __name__ == '__main__': - run_tests(100000) -- cgit v1.2.1 From 2c081f9a4af8928505ce4ea6ca2747ccb2e649c7 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 3 Sep 2014 19:30:38 -0400 Subject: - large resultsets --- examples/performance/__init__.py | 22 +++-- examples/performance/bulk_inserts.py | 35 ++++--- examples/performance/large_resultsets.py | 155 +++++++++++++++++++++++++++++++ examples/performance/single_inserts.py | 145 +++++++++++++++++++++++++++++ 4 files changed, 337 insertions(+), 20 deletions(-) create mode 100644 examples/performance/large_resultsets.py create mode 100644 examples/performance/single_inserts.py diff --git a/examples/performance/__init__.py b/examples/performance/__init__.py index ae914db96..b57f25b94 100644 --- a/examples/performance/__init__.py +++ b/examples/performance/__init__.py @@ -44,12 +44,12 @@ import os import time - class Profiler(object): tests = [] - def __init__(self, setup, options): + def __init__(self, options, setup=None, setup_once=None): self.setup = setup + self.setup_once = setup_once self.test = options.test self.dburl = options.dburl self.runsnake = options.runsnake @@ -72,6 +72,9 @@ class Profiler(object): else: tests = self.tests + if self.setup_once: + print("Running setup once...") + self.setup_once(self.dburl, self.echo, self.num) print("Tests to run: %s" % ", ".join([t.__name__ for t in tests])) for test in tests: self._run_test(test) @@ -100,14 +103,15 @@ class Profiler(object): self.stats.append(TestResult(self, fn, total_time=total)) def _run_test(self, fn): - self.setup(self.dburl, self.echo) + if self.setup: + self.setup(self.dburl, self.echo, self.num) if self.profile or self.runsnake or self.dump: self._run_with_profile(fn) else: self._run_with_time(fn) @classmethod - def main(cls, setup): + def main(cls, num, setup=None, setup_once=None): parser = argparse.ArgumentParser() parser.add_argument( @@ -119,8 +123,9 @@ class Profiler(object): help="database URL, default sqlite:///profile.db" ) parser.add_argument( - '--num', type=int, default=100000, - help="Number of iterations/items/etc for tests, default 100000" + '--num', type=int, default=num, + help="Number of iterations/items/etc for tests; " + "default is %d module-specific" % num ) parser.add_argument( '--profile', action='store_true', @@ -133,13 +138,12 @@ class Profiler(object): help='invoke runsnakerun (implies --profile)') parser.add_argument( '--echo', action='store_true', - help="Echo SQL output" - ) + help="Echo SQL output") args = parser.parse_args() args.profile = args.profile or args.dump or args.runsnake - Profiler(setup, args).run() + Profiler(args, setup=setup, setup_once=setup_once).run() class TestResult(object): diff --git a/examples/performance/bulk_inserts.py b/examples/performance/bulk_inserts.py index 42ab920a6..648d5f2aa 100644 --- a/examples/performance/bulk_inserts.py +++ b/examples/performance/bulk_inserts.py @@ -1,7 +1,7 @@ from . import Profiler from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy import Column, Integer, String, create_engine, bindparam from sqlalchemy.orm import Session Base = declarative_base() @@ -15,7 +15,7 @@ class Customer(Base): description = Column(String(255)) -def setup_database(dburl, echo): +def setup_database(dburl, echo, num): global engine engine = create_engine(dburl, echo=echo) Base.metadata.drop_all(engine) @@ -111,22 +111,35 @@ def test_core_insert(n): @Profiler.profile -def test_sqlite_raw(n): - """pysqlite's pure C API inserting rows in bulk, no pure Python at all""" - conn = engine.raw_connection() +def test_dbapi_raw(n): + """The DBAPI's pure C API inserting rows in bulk, no pure Python at all""" + + conn = engine.pool._creator() cursor = conn.cursor() - cursor.executemany( - "INSERT INTO customer (name, description) VALUES(:name, :description)", - [ + compiled = Customer.__table__.insert().values( + name=bindparam('name'), + description=bindparam('description')).\ + compile(dialect=engine.dialect) + + if compiled.positional: + args = ( + ('customer name %d' % i, 'customer description %d' % i) + for i in range(n)) + else: + args = ( dict( name='customer name %d' % i, description='customer description %d' % i ) for i in range(n) - ] + ) + + cursor.executemany( + str(compiled), + list(args) ) conn.commit() - + conn.close() if __name__ == '__main__': - Profiler.main(setup=setup_database) + Profiler.main(setup=setup_database, num=100000) diff --git a/examples/performance/large_resultsets.py b/examples/performance/large_resultsets.py new file mode 100644 index 000000000..268c6dc87 --- /dev/null +++ b/examples/performance/large_resultsets.py @@ -0,0 +1,155 @@ +"""In this series of tests, we are looking at time to load 1M very small +and simple rows. + +""" +from . import Profiler + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, create_engine, literal_column +from sqlalchemy.orm import Session, Bundle + +Base = declarative_base() +engine = None + + +class Customer(Base): + __tablename__ = "customer" + id = Column(Integer, primary_key=True) + name = Column(String(255)) + description = Column(String(255)) + + +def setup_database(dburl, echo, num): + global engine + engine = create_engine(dburl, echo=echo) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + s = Session(engine) + for chunk in range(0, num, 10000): + s.bulk_insert_mappings(Customer, [ + { + 'name': 'customer name %d' % i, + 'description': 'customer description %d' % i + } for i in range(chunk, chunk + 10000) + ]) + s.commit() + + +@Profiler.profile +def test_orm_full_objects(n): + """Load fully tracked objects using the ORM.""" + + sess = Session(engine) + # avoid using all() so that we don't have the overhead of building + # a large list of full objects in memory + for obj in sess.query(Customer).yield_per(1000).limit(n): + pass + + +@Profiler.profile +def test_orm_bundles(n): + """Load lightweight "bundle" objects using the ORM.""" + + sess = Session(engine) + bundle = Bundle('customer', + Customer.id, Customer.name, Customer.description) + for row in sess.query(bundle).yield_per(10000).limit(n): + pass + + +@Profiler.profile +def test_orm_columns(n): + """Load individual columns into named tuples using the ORM.""" + + sess = Session(engine) + for row in sess.query( + Customer.id, Customer.name, + Customer.description).yield_per(10000).limit(n): + pass + + +@Profiler.profile +def test_core_fetchall(n): + """Load Core result rows using Core / fetchall.""" + + with engine.connect() as conn: + result = conn.execute(Customer.__table__.select().limit(n)).fetchall() + for row in result: + data = row['id'], row['name'], row['description'] + + +@Profiler.profile +def test_core_fetchchunks_w_streaming(n): + """Load Core result rows using Core with fetchmany and + streaming results.""" + + with engine.connect() as conn: + result = conn.execution_options(stream_results=True).\ + execute(Customer.__table__.select().limit(n)) + while True: + chunk = result.fetchmany(10000) + if not chunk: + break + for row in chunk: + data = row['id'], row['name'], row['description'] + + +@Profiler.profile +def test_core_fetchchunks(n): + """Load Core result rows using Core / fetchmany.""" + + with engine.connect() as conn: + result = conn.execute(Customer.__table__.select().limit(n)) + while True: + chunk = result.fetchmany(10000) + if not chunk: + break + for row in chunk: + data = row['id'], row['name'], row['description'] + + +@Profiler.profile +def test_dbapi_fetchall(n): + """Load DBAPI cursor rows using fetchall()""" + + _test_dbapi_raw(n, True) + + +@Profiler.profile +def test_dbapi_fetchchunks(n): + """Load DBAPI cursor rows using fetchmany() + (usually doesn't limit memory)""" + + _test_dbapi_raw(n, False) + + +def _test_dbapi_raw(n, fetchall): + compiled = Customer.__table__.select().limit(n).\ + compile( + dialect=engine.dialect, + compile_kwargs={"literal_binds": True}) + + sql = str(compiled) + + import pdb + pdb.set_trace() + conn = engine.raw_connection() + cursor = conn.cursor() + cursor.execute(sql) + + if fetchall: + for row in cursor.fetchall(): + # ensure that we fully fetch! + data = row[0], row[1], row[2] + else: + while True: + chunk = cursor.fetchmany(10000) + if not chunk: + break + for row in chunk: + data = row[0], row[1], row[2] + conn.close() + +if __name__ == '__main__': + Profiler.main(setup_once=setup_database, num=1000000) diff --git a/examples/performance/single_inserts.py b/examples/performance/single_inserts.py new file mode 100644 index 000000000..671bbbe9c --- /dev/null +++ b/examples/performance/single_inserts.py @@ -0,0 +1,145 @@ +"""In this series of tests, we're looking at a method that inserts a row +within a distinct transaction, and afterwards returns to essentially a +"closed" state. This would be analogous to an API call that starts up +a database connection, inserts the row, commits and closes. + +""" +from . import Profiler + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, create_engine, bindparam, pool +from sqlalchemy.orm import Session + +Base = declarative_base() +engine = None + + +class Customer(Base): + __tablename__ = "customer" + id = Column(Integer, primary_key=True) + name = Column(String(255)) + description = Column(String(255)) + + +def setup_database(dburl, echo, num): + global engine + engine = create_engine(dburl, echo=echo) + if engine.dialect.name == 'sqlite': + engine.pool = pool.StaticPool(creator=engine.pool._creator) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + +@Profiler.profile +def test_orm_commit(n): + """Individual INSERT/COMMIT pairs via the ORM""" + + for i in range(n): + session = Session(bind=engine) + session.add( + Customer( + name='customer name %d' % i, + description='customer description %d' % i) + ) + session.commit() + + +@Profiler.profile +def test_bulk_save(n): + """Individual INSERT/COMMIT pairs using the "bulk" API """ + + for i in range(n): + session = Session(bind=engine) + session.bulk_save_objects([ + Customer( + name='customer name %d' % i, + description='customer description %d' % i + )]) + session.commit() + + +@Profiler.profile +def test_bulk_insert_dictionaries(n): + """Individual INSERT/COMMIT pairs using the "bulk" API with dictionaries""" + + for i in range(n): + session = Session(bind=engine) + session.bulk_insert_mappings(Customer, [ + dict( + name='customer name %d' % i, + description='customer description %d' % i + )]) + session.commit() + + +@Profiler.profile +def test_core(n): + """Individual INSERT/COMMIT pairs using Core.""" + + for i in range(n): + with engine.begin() as conn: + conn.execute( + Customer.__table__.insert(), + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + ) + + +@Profiler.profile +def test_dbapi_raw_w_connect(n): + """Individual INSERT/COMMIT pairs using a pure DBAPI connection, + connect each time.""" + + _test_dbapi_raw(n, True) + + +@Profiler.profile +def test_dbapi_raw_w_pool(n): + """Individual INSERT/COMMIT pairs using a pure DBAPI connection, + using a connection pool.""" + + _test_dbapi_raw(n, False) + + +def _test_dbapi_raw(n, connect): + compiled = Customer.__table__.insert().values( + name=bindparam('name'), + description=bindparam('description')).\ + compile(dialect=engine.dialect) + + if compiled.positional: + args = ( + ('customer name %d' % i, 'customer description %d' % i) + for i in range(n)) + else: + args = ( + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + for i in range(n) + ) + sql = str(compiled) + + if connect: + for arg in args: + # there's no connection pool, so if these were distinct + # calls, we'd be connecting each time + conn = engine.pool._creator() + cursor = conn.cursor() + cursor.execute(sql, arg) + conn.commit() + conn.close() + else: + for arg in args: + conn = engine.raw_connection() + cursor = conn.cursor() + cursor.execute(sql, arg) + conn.commit() + conn.close() + + +if __name__ == '__main__': + Profiler.main(setup=setup_database, num=10000) -- cgit v1.2.1 From cbef6a7d58ee42e33167a14e6a31a124aa0bf08e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 3 Sep 2014 20:07:08 -0400 Subject: refine --- examples/performance/large_resultsets.py | 66 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/examples/performance/large_resultsets.py b/examples/performance/large_resultsets.py index 7383db734..c9ce23d61 100644 --- a/examples/performance/large_resultsets.py +++ b/examples/performance/large_resultsets.py @@ -1,11 +1,22 @@ -"""In this series of tests, we are looking at time to load 1M very small -and simple rows. +"""In this series of tests, we are looking at time to load a large number +of very small and simple rows. + +A special test here illustrates the difference between fetching the +rows from the raw DBAPI and throwing them away, vs. assembling each +row into a completely basic Python object and appending to a list. The +time spent typically more than doubles. The point is that while +DBAPIs will give you raw rows very fast if they are written in C, the +moment you do anything with those rows, even something trivial, +overhead grows extremely fast in cPython. SQLAlchemy's Core and +lighter-weight ORM options add absolutely minimal overhead, and the +full blown ORM doesn't do terribly either even though mapped objects +provide a huge amount of functionality. """ from . import Profiler from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, Integer, String, create_engine, literal_column +from sqlalchemy import Column, Integer, String, create_engine from sqlalchemy.orm import Session, Bundle Base = declarative_base() @@ -71,7 +82,7 @@ def test_orm_columns(n): @Profiler.profile def test_core_fetchall(n): - """Load Core result rows using Core / fetchall.""" + """Load Core result rows using fetchall.""" with engine.connect() as conn: result = conn.execute(Customer.__table__.select().limit(n)).fetchall() @@ -80,9 +91,8 @@ def test_core_fetchall(n): @Profiler.profile -def test_core_fetchchunks_w_streaming(n): - """Load Core result rows using Core with fetchmany and - streaming results.""" +def test_core_fetchmany_w_streaming(n): + """Load Core result rows using fetchmany/streaming.""" with engine.connect() as conn: result = conn.execution_options(stream_results=True).\ @@ -96,7 +106,7 @@ def test_core_fetchchunks_w_streaming(n): @Profiler.profile -def test_core_fetchchunks(n): +def test_core_fetchmany(n): """Load Core result rows using Core / fetchmany.""" with engine.connect() as conn: @@ -110,44 +120,54 @@ def test_core_fetchchunks(n): @Profiler.profile -def test_dbapi_fetchall(n): - """Load DBAPI cursor rows using fetchall()""" +def test_dbapi_fetchall_plus_append_objects(n): + """Load rows using DBAPI fetchall(), make a list of objects.""" _test_dbapi_raw(n, True) @Profiler.profile -def test_dbapi_fetchchunks(n): - """Load DBAPI cursor rows using fetchmany() - (usually doesn't limit memory)""" +def test_dbapi_fetchall_no_object(n): + """Load rows using DBAPI fetchall(), don't make any objects.""" _test_dbapi_raw(n, False) -def _test_dbapi_raw(n, fetchall): +def _test_dbapi_raw(n, make_objects): compiled = Customer.__table__.select().limit(n).\ compile( dialect=engine.dialect, compile_kwargs={"literal_binds": True}) + if make_objects: + # because if you're going to roll your own, you're probably + # going to do this, so see how this pushes you right back into + # ORM land anyway :) + class SimpleCustomer(object): + def __init__(self, id, name, description): + self.id = id + self.name = name + self.description = description + sql = str(compiled) conn = engine.raw_connection() cursor = conn.cursor() cursor.execute(sql) - if fetchall: + if make_objects: + result = [] for row in cursor.fetchall(): # ensure that we fully fetch! - data = row[0], row[1], row[2] + customer = SimpleCustomer( + id=row[0], name=row[1], description=row[2]) + result.append(customer) else: - while True: - chunk = cursor.fetchmany(10000) - if not chunk: - break - for row in chunk: - data = row[0], row[1], row[2] + for row in cursor.fetchall(): + # ensure that we fully fetch! + data = row[0], row[1], row[2] + conn.close() if __name__ == '__main__': - Profiler.main(setup_once=setup_database, num=1000000) + Profiler.main(setup_once=setup_database, num=500000) -- cgit v1.2.1 From eb81531275c07a0ab8c74eadc7881cfcff27ba21 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 3 Sep 2014 20:30:52 -0400 Subject: tweak --- examples/performance/bulk_inserts.py | 11 ++++++++--- examples/performance/large_resultsets.py | 4 +--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/performance/bulk_inserts.py b/examples/performance/bulk_inserts.py index 648d5f2aa..531003aa6 100644 --- a/examples/performance/bulk_inserts.py +++ b/examples/performance/bulk_inserts.py @@ -1,3 +1,8 @@ +"""This series of tests illustrates different ways to INSERT a large number +of rows in bulk. + + +""" from . import Profiler from sqlalchemy.ext.declarative import declarative_base @@ -69,7 +74,7 @@ def test_flush_pk_given(n): @Profiler.profile def test_bulk_save(n): - """Batched INSERT statements via the ORM in "bulk", discarding PK values.""" + """Batched INSERT statements via the ORM in "bulk", discarding PKs.""" session = Session(bind=engine) session.bulk_save_objects([ Customer( @@ -83,7 +88,7 @@ def test_bulk_save(n): @Profiler.profile def test_bulk_insert_mappings(n): - """Batched INSERT statements via the ORM "bulk", using dictionaries instead of objects""" + """Batched INSERT statements via the ORM "bulk", using dictionaries.""" session = Session(bind=engine) session.bulk_insert_mappings(Customer, [ dict( @@ -112,7 +117,7 @@ def test_core_insert(n): @Profiler.profile def test_dbapi_raw(n): - """The DBAPI's pure C API inserting rows in bulk, no pure Python at all""" + """The DBAPI's API inserting rows in bulk.""" conn = engine.pool._creator() cursor = conn.cursor() diff --git a/examples/performance/large_resultsets.py b/examples/performance/large_resultsets.py index c9ce23d61..77c0246fc 100644 --- a/examples/performance/large_resultsets.py +++ b/examples/performance/large_resultsets.py @@ -121,7 +121,7 @@ def test_core_fetchmany(n): @Profiler.profile def test_dbapi_fetchall_plus_append_objects(n): - """Load rows using DBAPI fetchall(), make a list of objects.""" + """Load rows using DBAPI fetchall(), generate an object for each row.""" _test_dbapi_raw(n, True) @@ -156,12 +156,10 @@ def _test_dbapi_raw(n, make_objects): cursor.execute(sql) if make_objects: - result = [] for row in cursor.fetchall(): # ensure that we fully fetch! customer = SimpleCustomer( id=row[0], name=row[1], description=row[2]) - result.append(customer) else: for row in cursor.fetchall(): # ensure that we fully fetch! -- cgit v1.2.1 From d2c05c36a5c5f5b4838e924b4de2280f73916c99 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Sep 2014 20:55:38 -0400 Subject: - add a test that shows query caching. --- examples/performance/single_inserts.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/performance/single_inserts.py b/examples/performance/single_inserts.py index 671bbbe9c..4178ccea8 100644 --- a/examples/performance/single_inserts.py +++ b/examples/performance/single_inserts.py @@ -87,6 +87,23 @@ def test_core(n): ) +@Profiler.profile +def test_core_query_caching(n): + """Individual INSERT/COMMIT pairs using Core with query caching""" + + cache = {} + ins = Customer.__table__.insert() + for i in range(n): + with engine.begin() as conn: + conn.execution_options(compiled_cache=cache).execute( + ins, + dict( + name='customer name %d' % i, + description='customer description %d' % i + ) + ) + + @Profiler.profile def test_dbapi_raw_w_connect(n): """Individual INSERT/COMMIT pairs using a pure DBAPI connection, @@ -130,6 +147,7 @@ def _test_dbapi_raw(n, connect): conn = engine.pool._creator() cursor = conn.cursor() cursor.execute(sql, arg) + lastrowid = cursor.lastrowid conn.commit() conn.close() else: @@ -137,6 +155,7 @@ def _test_dbapi_raw(n, connect): conn = engine.raw_connection() cursor = conn.cursor() cursor.execute(sql, arg) + lastrowid = cursor.lastrowid conn.commit() conn.close() -- cgit v1.2.1 From fa7c8f88113d2e769274dee4aa4247b9c9aadec8 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 6 Sep 2014 13:01:21 -0400 Subject: - try to finish up the performance example for now --- doc/build/orm/examples.rst | 7 + examples/performance/__init__.py | 299 +++++++++++++++++++++++++++---- examples/performance/__main__.py | 5 + examples/performance/bulk_inserts.py | 6 +- examples/performance/large_resultsets.py | 6 +- examples/performance/single_inserts.py | 12 +- 6 files changed, 291 insertions(+), 44 deletions(-) create mode 100644 examples/performance/__main__.py diff --git a/doc/build/orm/examples.rst b/doc/build/orm/examples.rst index b820dba9f..93478381a 100644 --- a/doc/build/orm/examples.rst +++ b/doc/build/orm/examples.rst @@ -62,6 +62,13 @@ Nested Sets .. automodule:: examples.nested_sets +.. _examples_performance: + +Performance +----------- + +.. automodule:: examples.performance + .. _examples_relationships: Relationship Join Conditions diff --git a/examples/performance/__init__.py b/examples/performance/__init__.py index b57f25b94..6e2e1fc89 100644 --- a/examples/performance/__init__.py +++ b/examples/performance/__init__.py @@ -1,55 +1,232 @@ """A performance profiling suite for a variety of SQLAlchemy use cases. -The suites here each focus on some specific type of use case, one which -has a particular performance profile: +Each suite focuses on a specific use case with a particular performance +profile and associated implications: * bulk inserts * individual inserts, with or without transactions * fetching large numbers of rows -* running lots of small queries +* running lots of small queries (TODO) -All suites include a variety of use patterns with both the Core and -ORM, and are sorted in order of performance from worst to greatest, -inversely based on amount of functionality provided by SQLAlchemy, +All suites include a variety of use patterns illustrating both Core +and ORM use, and are generally sorted in order of performance from worst +to greatest, inversely based on amount of functionality provided by SQLAlchemy, greatest to least (these two things generally correspond perfectly). -Each suite is run as a module, and provides a consistent command line -interface:: +A command line tool is presented at the package level which allows +individual suites to be run:: - $ python -m examples.performance.bulk_inserts --profile --num 1000 + $ python -m examples.performance --help + usage: python -m examples.performance [-h] [--test TEST] [--dburl DBURL] + [--num NUM] [--profile] [--dump] + [--runsnake] [--echo] -Using ``--help`` will allow all options:: + {bulk_inserts,large_resultsets,single_inserts} - $ python -m examples.performance.bulk_inserts --help -usage: bulk_inserts.py [-h] [--test TEST] [--dburl DBURL] [--num NUM] - [--profile] [--dump] [--runsnake] [--echo] + positional arguments: + {bulk_inserts,large_resultsets,single_inserts} + suite to run -optional arguments: - -h, --help show this help message and exit - --test TEST run specific test name - --dburl DBURL database URL, default sqlite:///profile.db - --num NUM Number of iterations/items/etc for tests, default 100000 - --profile run profiling and dump call counts - --dump dump full call profile (implies --profile) - --runsnake invoke runsnakerun (implies --profile) - --echo Echo SQL output + optional arguments: + -h, --help show this help message and exit + --test TEST run specific test name + --dburl DBURL database URL, default sqlite:///profile.db + --num NUM Number of iterations/items/etc for tests; default is 0 + module-specific + --profile run profiling and dump call counts + --dump dump full call profile (implies --profile) + --runsnake invoke runsnakerun (implies --profile) + --echo Echo SQL output +An example run looks like:: + + $ python -m examples.performance bulk_inserts + +Or with options:: + + $ python -m examples.performance bulk_inserts \\ + --dburl mysql+mysqldb://scott:tiger@localhost/test \\ + --profile --num 1000 + + +Running all tests with time +--------------------------- + +This is the default form of run:: + + $ python -m examples.performance single_inserts + Tests to run: test_orm_commit, test_bulk_save, + test_bulk_insert_dictionaries, test_core, + test_core_query_caching, test_dbapi_raw_w_connect, + test_dbapi_raw_w_pool + + test_orm_commit : Individual INSERT/COMMIT pairs via the + ORM (10000 iterations); total time 13.690218 sec + test_bulk_save : Individual INSERT/COMMIT pairs using + the "bulk" API (10000 iterations); total time 11.290371 sec + test_bulk_insert_dictionaries : Individual INSERT/COMMIT pairs using + the "bulk" API with dictionaries (10000 iterations); + total time 10.814626 sec + test_core : Individual INSERT/COMMIT pairs using Core. + (10000 iterations); total time 9.665620 sec + test_core_query_caching : Individual INSERT/COMMIT pairs using Core + with query caching (10000 iterations); total time 9.209010 sec + test_dbapi_raw_w_connect : Individual INSERT/COMMIT pairs w/ DBAPI + + connection each time (10000 iterations); total time 9.551103 sec + test_dbapi_raw_w_pool : Individual INSERT/COMMIT pairs w/ DBAPI + + connection pool (10000 iterations); total time 8.001813 sec + +Dumping Profiles for Individual Tests +-------------------------------------- + +A Python profile output can be dumped for all tests, or more commonly +individual tests:: + + $ python -m examples.performance single_inserts --test test_core --num 1000 --dump + Tests to run: test_core + test_core : Individual INSERT/COMMIT pairs using Core. (1000 iterations); total fn calls 186109 + 186109 function calls (186102 primitive calls) in 1.089 seconds + + Ordered by: internal time, call count + + ncalls tottime percall cumtime percall filename:lineno(function) + 1000 0.634 0.001 0.634 0.001 {method 'commit' of 'sqlite3.Connection' objects} + 1000 0.154 0.000 0.154 0.000 {method 'execute' of 'sqlite3.Cursor' objects} + 1000 0.021 0.000 0.074 0.000 /Users/classic/dev/sqlalchemy/lib/sqlalchemy/sql/compiler.py:1950(_get_colparams) + 1000 0.015 0.000 0.034 0.000 /Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/default.py:503(_init_compiled) + 1 0.012 0.012 1.091 1.091 examples/performance/single_inserts.py:79(test_core) + + ... + +Using RunSnake +-------------- + +This option requires the `RunSnake `_ +command line tool be installed:: + + $ python -m examples.performance single_inserts --test test_core --num 1000 --runsnake + +A graphical RunSnake output will be displayed. + +.. _examples_profiling_writeyourown: + +Writing your Own Suites +----------------------- + +The profiler suite system is extensible, and can be applied to your own set +of tests. This is a valuable technique to use in deciding upon the proper +approach for some performance-critical set of routines. For example, +if we wanted to profile the difference between several kinds of loading, +we can create a file ``test_loads.py``, with the following content:: + + from examples.performance import Profiler + from sqlalchemy import Integer, Column, create_engine, ForeignKey + from sqlalchemy.orm import relationship, joinedload, subqueryload, Session + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + engine = None + session = None + + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + children = relationship("Child") + + + class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('parent.id')) + + + # Init with name of file, default number of items + Profiler.init("test_loads", 1000) + + + @Profiler.setup_once + def setup_once(dburl, echo, num): + "setup once. create an engine, insert fixture data" + global engine + engine = create_engine(dburl, echo=echo) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + sess = Session(engine) + sess.add_all([ + Parent(children=[Child() for j in range(100)]) + for i in range(num) + ]) + sess.commit() + + + @Profiler.setup + def setup(dburl, echo, num): + "setup per test. create a new Session." + global session + session = Session(engine) + # pre-connect so this part isn't profiled (if we choose) + session.connection() + + + @Profiler.profile + def test_lazyload(n): + "load everything, no eager loading." + + for parent in session.query(Parent): + parent.children + + + @Profiler.profile + def test_joinedload(n): + "load everything, joined eager loading." + + for parent in session.query(Parent).options(joinedload("children")): + parent.children + + + @Profiler.profile + def test_subqueryload(n): + "load everything, subquery eager loading." + + for parent in session.query(Parent).options(subqueryload("children")): + parent.children + + if __name__ == '__main__': + Profiler.main() + +We can run our new script directly:: + + $ python test_loads.py --dburl postgresql+psycopg2://scott:tiger@localhost/test + Running setup once... + Tests to run: test_lazyload, test_joinedload, test_subqueryload + test_lazyload : load everything, no eager loading. (1000 iterations); total time 11.971159 sec + test_joinedload : load everything, joined eager loading. (1000 iterations); total time 2.754592 sec + test_subqueryload : load everything, subquery eager loading. (1000 iterations); total time 2.977696 sec + +As well as see RunSnake output for an individual test:: + + $ python test_loads.py --num 100 --runsnake --test test_joinedload """ import argparse import cProfile -import StringIO import pstats import os import time +import re +import sys class Profiler(object): tests = [] - def __init__(self, options, setup=None, setup_once=None): - self.setup = setup - self.setup_once = setup_once + _setup = None + _setup_once = None + name = None + num = 0 + + def __init__(self, options): self.test = options.test self.dburl = options.dburl self.runsnake = options.runsnake @@ -59,11 +236,34 @@ class Profiler(object): self.echo = options.echo self.stats = [] + @classmethod + def init(cls, name, num): + cls.name = name + cls.num = num + @classmethod def profile(cls, fn): + if cls.name is None: + raise ValueError( + "Need to call Profile.init(, ) first.") cls.tests.append(fn) return fn + @classmethod + def setup(cls, fn): + if cls._setup is not None: + raise ValueError("setup function already set to %s" % cls._setup) + cls._setup = staticmethod(fn) + return fn + + @classmethod + def setup_once(cls, fn): + if cls._setup_once is not None: + raise ValueError( + "setup_once function already set to %s" % cls._setup_once) + cls._setup_once = staticmethod(fn) + return fn + def run(self): if self.test: tests = [fn for fn in self.tests if fn.__name__ == self.test] @@ -72,9 +272,9 @@ class Profiler(object): else: tests = self.tests - if self.setup_once: + if self._setup_once: print("Running setup once...") - self.setup_once(self.dburl, self.echo, self.num) + self._setup_once(self.dburl, self.echo, self.num) print("Tests to run: %s" % ", ".join([t.__name__ for t in tests])) for test in tests: self._run_test(test) @@ -88,8 +288,7 @@ class Profiler(object): finally: pr.disable() - output = StringIO.StringIO() - stats = pstats.Stats(pr, stream=output).sort_stats('cumulative') + stats = pstats.Stats(pr).sort_stats('cumulative') self.stats.append(TestResult(self, fn, stats=stats)) return result @@ -103,29 +302,42 @@ class Profiler(object): self.stats.append(TestResult(self, fn, total_time=total)) def _run_test(self, fn): - if self.setup: - self.setup(self.dburl, self.echo, self.num) + if self._setup: + self._setup(self.dburl, self.echo, self.num) if self.profile or self.runsnake or self.dump: self._run_with_profile(fn) else: self._run_with_time(fn) @classmethod - def main(cls, num, setup=None, setup_once=None): - parser = argparse.ArgumentParser() + def main(cls): + + parser = argparse.ArgumentParser("python -m examples.performance") + + if cls.name is None: + parser.add_argument( + "name", choices=cls._suite_names(), help="suite to run") + + if len(sys.argv) > 1: + potential_name = sys.argv[1] + try: + suite = __import__(__name__ + "." + potential_name) + except ImportError: + pass parser.add_argument( "--test", type=str, help="run specific test name" ) + parser.add_argument( '--dburl', type=str, default="sqlite:///profile.db", help="database URL, default sqlite:///profile.db" ) parser.add_argument( - '--num', type=int, default=num, + '--num', type=int, default=cls.num, help="Number of iterations/items/etc for tests; " - "default is %d module-specific" % num + "default is %d module-specific" % cls.num ) parser.add_argument( '--profile', action='store_true', @@ -143,7 +355,19 @@ class Profiler(object): args.profile = args.profile or args.dump or args.runsnake - Profiler(args, setup=setup, setup_once=setup_once).run() + if cls.name is None: + suite = __import__(__name__ + "." + args.name) + + Profiler(args).run() + + @classmethod + def _suite_names(cls): + suites = [] + for file_ in os.listdir(os.path.dirname(__file__)): + match = re.match(r'^([a-z].*).py$', file_) + if match: + suites.append(match.group(1)) + return suites class TestResult(object): @@ -185,3 +409,4 @@ class TestResult(object): finally: os.remove(filename) + diff --git a/examples/performance/__main__.py b/examples/performance/__main__.py new file mode 100644 index 000000000..957d6c699 --- /dev/null +++ b/examples/performance/__main__.py @@ -0,0 +1,5 @@ +from . import Profiler + +if __name__ == '__main__': + Profiler.main() + diff --git a/examples/performance/bulk_inserts.py b/examples/performance/bulk_inserts.py index 531003aa6..9c3cff5b2 100644 --- a/examples/performance/bulk_inserts.py +++ b/examples/performance/bulk_inserts.py @@ -20,6 +20,10 @@ class Customer(Base): description = Column(String(255)) +Profiler.init("bulk_inserts", num=100000) + + +@Profiler.setup def setup_database(dburl, echo, num): global engine engine = create_engine(dburl, echo=echo) @@ -147,4 +151,4 @@ def test_dbapi_raw(n): conn.close() if __name__ == '__main__': - Profiler.main(setup=setup_database, num=100000) + Profiler.main() diff --git a/examples/performance/large_resultsets.py b/examples/performance/large_resultsets.py index 77c0246fc..0a5857b75 100644 --- a/examples/performance/large_resultsets.py +++ b/examples/performance/large_resultsets.py @@ -30,6 +30,10 @@ class Customer(Base): description = Column(String(255)) +Profiler.init("large_resultsets", num=500000) + + +@Profiler.setup_once def setup_database(dburl, echo, num): global engine engine = create_engine(dburl, echo=echo) @@ -168,4 +172,4 @@ def _test_dbapi_raw(n, make_objects): conn.close() if __name__ == '__main__': - Profiler.main(setup_once=setup_database, num=500000) + Profiler.main() diff --git a/examples/performance/single_inserts.py b/examples/performance/single_inserts.py index 4178ccea8..cfce90300 100644 --- a/examples/performance/single_inserts.py +++ b/examples/performance/single_inserts.py @@ -21,6 +21,10 @@ class Customer(Base): description = Column(String(255)) +Profiler.init("single_inserts", num=10000) + + +@Profiler.setup def setup_database(dburl, echo, num): global engine engine = create_engine(dburl, echo=echo) @@ -106,16 +110,14 @@ def test_core_query_caching(n): @Profiler.profile def test_dbapi_raw_w_connect(n): - """Individual INSERT/COMMIT pairs using a pure DBAPI connection, - connect each time.""" + """Individual INSERT/COMMIT pairs w/ DBAPI + connection each time""" _test_dbapi_raw(n, True) @Profiler.profile def test_dbapi_raw_w_pool(n): - """Individual INSERT/COMMIT pairs using a pure DBAPI connection, - using a connection pool.""" + """Individual INSERT/COMMIT pairs w/ DBAPI + connection pool""" _test_dbapi_raw(n, False) @@ -161,4 +163,4 @@ def _test_dbapi_raw(n, connect): if __name__ == '__main__': - Profiler.main(setup=setup_database, num=10000) + Profiler.main() -- cgit v1.2.1 From b9d430af752b7cc955932a54a8f8db18f46d89a6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 16 Sep 2014 11:57:03 -0400 Subject: - add differentiating examples of list() vs. iteration --- examples/performance/large_resultsets.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/performance/large_resultsets.py b/examples/performance/large_resultsets.py index 0a5857b75..fbe77c759 100644 --- a/examples/performance/large_resultsets.py +++ b/examples/performance/large_resultsets.py @@ -52,12 +52,18 @@ def setup_database(dburl, echo, num): @Profiler.profile -def test_orm_full_objects(n): - """Load fully tracked objects using the ORM.""" +def test_orm_full_objects_list(n): + """Load fully tracked ORM objects into one big list().""" + + sess = Session(engine) + objects = list(sess.query(Customer).limit(n)) + + +@Profiler.profile +def test_orm_full_objects_chunks(n): + """Load fully tracked ORM objects a chunk at a time using yield_per().""" sess = Session(engine) - # avoid using all() so that we don't have the overhead of building - # a large list of full objects in memory for obj in sess.query(Customer).yield_per(1000).limit(n): pass -- cgit v1.2.1 From 0c19d765dce89970c0395f57f15eb5b0f09c2a29 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 6 Nov 2014 17:29:22 -0500 Subject: bulk_updates --- examples/performance/bulk_updates.py | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 examples/performance/bulk_updates.py diff --git a/examples/performance/bulk_updates.py b/examples/performance/bulk_updates.py new file mode 100644 index 000000000..9522e4bf5 --- /dev/null +++ b/examples/performance/bulk_updates.py @@ -0,0 +1,54 @@ +"""This series of tests illustrates different ways to UPDATE a large number +of rows in bulk. + + +""" +from . import Profiler + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, create_engine, bindparam +from sqlalchemy.orm import Session + +Base = declarative_base() +engine = None + + +class Customer(Base): + __tablename__ = "customer" + id = Column(Integer, primary_key=True) + name = Column(String(255)) + description = Column(String(255)) + + +Profiler.init("bulk_updates", num=100000) + + +@Profiler.setup +def setup_database(dburl, echo, num): + global engine + engine = create_engine(dburl, echo=echo) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + s = Session(engine) + for chunk in range(0, num, 10000): + s.bulk_insert_mappings(Customer, [ + { + 'name': 'customer name %d' % i, + 'description': 'customer description %d' % i + } for i in range(chunk, chunk + 10000) + ]) + s.commit() + + +@Profiler.profile +def test_orm_flush(n): + """UPDATE statements via the ORM flush process.""" + session = Session(bind=engine) + for chunk in range(0, n, 1000): + customers = session.query(Customer).\ + filter(Customer.id.between(chunk, chunk + 1000)).all() + for customer in customers: + customer.description += "updated" + session.flush() + session.commit() -- cgit v1.2.1 From 9fb6acad67145d0b0b973a7a074eb5b2baf45086 Mon Sep 17 00:00:00 2001 From: "Ryan P. Kelly" Date: Thu, 20 Nov 2014 14:40:32 -0500 Subject: Report the type of unexpected expression objects --- lib/sqlalchemy/sql/elements.py | 3 ++- test/sql/test_defaults.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index 734f78632..4c66061c8 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -3732,7 +3732,8 @@ def _literal_as_text(element, warn=False): return _const_expr(element) else: raise exc.ArgumentError( - "SQL expression object or string expected." + "SQL expression object or string expected, got object of type %r " + "instead" % type(element) ) diff --git a/test/sql/test_defaults.py b/test/sql/test_defaults.py index 10e557b76..21d04ea38 100644 --- a/test/sql/test_defaults.py +++ b/test/sql/test_defaults.py @@ -368,7 +368,8 @@ class DefaultTest(fixtures.TestBase): ): assert_raises_message( sa.exc.ArgumentError, - "SQL expression object or string expected.", + "SQL expression object or string expected, got object of type " + " instead", t.select, [const] ) assert_raises_message( -- cgit v1.2.1 From 0e61acaf145f57c78a13fc5c20052e24472cfb02 Mon Sep 17 00:00:00 2001 From: Sebastian Bank Date: Thu, 4 Dec 2014 14:34:08 +0100 Subject: warn on duplicate polymorphic_identity --- lib/sqlalchemy/orm/mapper.py | 9 +++++++++ test/orm/test_mapper.py | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 863dab5cb..0e0c1a833 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -974,6 +974,15 @@ class Mapper(InspectionAttr): self._all_tables = self.inherits._all_tables if self.polymorphic_identity is not None: + if self.polymorphic_identity in self.polymorphic_map: + util.warn( + "Reassigning polymorphic association for identity %r " + "from %r to %r: Check for duplicate use of %r as " + "value for polymorphic_identity." % + (self.polymorphic_identity, + self.polymorphic_map[self.polymorphic_identity], + self, self.polymorphic_identity) + ) self.polymorphic_map[self.polymorphic_identity] = self else: diff --git a/test/orm/test_mapper.py b/test/orm/test_mapper.py index 63ba1a207..264b386d4 100644 --- a/test/orm/test_mapper.py +++ b/test/orm/test_mapper.py @@ -716,6 +716,19 @@ class MapperTest(_fixtures.FixtureTest, AssertsCompiledSQL): m3.identity_key_from_instance(AddressUser()) ) + def test_reassign_polymorphic_identity_warns(self): + User = self.classes.User + users = self.tables.users + class MyUser(User): + pass + m1 = mapper(User, users, polymorphic_on=users.c.name, + polymorphic_identity='user') + assert_raises_message( + sa.exc.SAWarning, + "Reassigning polymorphic association for identity 'user'", + mapper, + MyUser, users, inherits=User, polymorphic_identity='user' + ) def test_illegal_non_primary(self): -- cgit v1.2.1 From 87bfcf91e9659893f17adf307090bc0a4a8a8f23 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Dec 2014 12:01:19 -0500 Subject: - The :meth:`.PGDialect.has_table` method will now query against ``pg_catalog.pg_table_is_visible(c.oid)``, rather than testing for an exact schema match, when the schema name is None; this so that the method will also illustrate that temporary tables are present. Note that this is a behavioral change, as Postgresql allows a non-temporary table to silently overwrite an existing temporary table of the same name, so this changes the behavior of ``checkfirst`` in that unusual scenario. fixes #3264 --- doc/build/changelog/changelog_10.rst | 17 ++++++++ doc/build/changelog/migration_10.rst | 58 +++++++++++++++++++++++++ lib/sqlalchemy/dialects/postgresql/base.py | 3 +- lib/sqlalchemy/testing/suite/test_reflection.py | 4 ++ test/dialect/postgresql/test_reflection.py | 12 +++++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index f2bd43a76..ad9eefa09 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -21,6 +21,23 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, postgresql + :tickets: 3264 + + The :meth:`.PGDialect.has_table` method will now query against + ``pg_catalog.pg_table_is_visible(c.oid)``, rather than testing + for an exact schema match, when the schema name is None; this + so that the method will also illustrate that temporary tables + are present. Note that this is a behavioral change, as Postgresql + allows a non-temporary table to silently overwrite an existing + temporary table of the same name, so this changes the behavior + of ``checkfirst`` in that unusual scenario. + + .. seealso:: + + :ref:`change_3264` + .. change:: :tags: bug, sql :tickets: 3260 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index c4157266b..e148e7d70 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -276,6 +276,64 @@ running 0.9 in production. :ticket:`2891` +.. _change_3264: + +Postgresql ``has_table()`` now works for temporary tables +--------------------------------------------------------- + +This is a simple fix such that "has table" for temporary tables now works, +so that code like the following may proceed:: + + from sqlalchemy import * + + metadata = MetaData() + user_tmp = Table( + "user_tmp", metadata, + Column("id", INT, primary_key=True), + Column('name', VARCHAR(50)), + prefixes=['TEMPORARY'] + ) + + e = create_engine("postgresql://scott:tiger@localhost/test", echo='debug') + with e.begin() as conn: + user_tmp.create(conn, checkfirst=True) + + # checkfirst will succeed + user_tmp.create(conn, checkfirst=True) + +The very unlikely case that this behavior will cause a non-failing application +to behave differently, is because Postgresql allows a non-temporary table +to silently overwrite a temporary table. So code like the following will +now act completely differently, no longer creating the real table following +the temporary table:: + + from sqlalchemy import * + + metadata = MetaData() + user_tmp = Table( + "user_tmp", metadata, + Column("id", INT, primary_key=True), + Column('name', VARCHAR(50)), + prefixes=['TEMPORARY'] + ) + + e = create_engine("postgresql://scott:tiger@localhost/test", echo='debug') + with e.begin() as conn: + user_tmp.create(conn, checkfirst=True) + + m2 = MetaData() + user = Table( + "user_tmp", m2, + Column("id", INT, primary_key=True), + Column('name', VARCHAR(50)), + ) + + # in 0.9, *will create* the new table, overwriting the old one. + # in 1.0, *will not create* the new table + user.create(conn, checkfirst=True) + +:ticket:`3264` + .. _feature_gh134: Postgresql FILTER keyword diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index baa640eaa..034ee9076 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1942,7 +1942,8 @@ class PGDialect(default.DefaultDialect): cursor = connection.execute( sql.text( "select relname from pg_class c join pg_namespace n on " - "n.oid=c.relnamespace where n.nspname=current_schema() " + "n.oid=c.relnamespace where " + "pg_catalog.pg_table_is_visible(c.oid) " "and relname=:name", bindparams=[ sql.bindparam('name', util.text_type(table_name), diff --git a/lib/sqlalchemy/testing/suite/test_reflection.py b/lib/sqlalchemy/testing/suite/test_reflection.py index 08b858b47..e58b6f068 100644 --- a/lib/sqlalchemy/testing/suite/test_reflection.py +++ b/lib/sqlalchemy/testing/suite/test_reflection.py @@ -128,6 +128,10 @@ class ComponentReflectionTest(fixtures.TablesTest): DDL("create temporary view user_tmp_v as " "select * from user_tmp") ) + event.listen( + user_tmp, "before_drop", + DDL("drop view user_tmp_v") + ) @classmethod def define_index(cls, metadata, users): diff --git a/test/dialect/postgresql/test_reflection.py b/test/dialect/postgresql/test_reflection.py index 8de71216e..0dda1fa45 100644 --- a/test/dialect/postgresql/test_reflection.py +++ b/test/dialect/postgresql/test_reflection.py @@ -322,6 +322,18 @@ class ReflectionTest(fixtures.TestBase): t2 = Table('t', m2, autoload=True) eq_([c.name for c in t2.primary_key], ['t_id']) + @testing.provide_metadata + def test_has_temporary_table(self): + assert not testing.db.has_table("some_temp_table") + user_tmp = Table( + "some_temp_table", self.metadata, + Column("id", Integer, primary_key=True), + Column('name', String(50)), + prefixes=['TEMPORARY'] + ) + user_tmp.create(testing.db) + assert testing.db.has_table("some_temp_table") + @testing.provide_metadata def test_cross_schema_reflection_one(self): -- cgit v1.2.1 From f5ff86983f9cc7914a89b96da1fd2638677d345b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Dec 2014 18:29:56 -0500 Subject: - The :meth:`.Operators.match` operator is now handled such that the return type is not strictly assumed to be boolean; it now returns a :class:`.Boolean` subclass called :class:`.MatchType`. The type will still produce boolean behavior when used in Python expressions, however the dialect can override its behavior at result time. In the case of MySQL, while the MATCH operator is typically used in a boolean context within an expression, if one actually queries for the value of a match expression, a floating point value is returned; this value is not compatible with SQLAlchemy's C-based boolean processor, so MySQL's result-set behavior now follows that of the :class:`.Float` type. A new operator object ``notmatch_op`` is also added to better allow dialects to define the negation of a match operation. fixes #3263 --- doc/build/changelog/changelog_10.rst | 23 +++++++++++++++++++ doc/build/changelog/migration_10.rst | 31 +++++++++++++++++++++++++ doc/build/core/types.rst | 3 +++ lib/sqlalchemy/dialects/mysql/base.py | 9 ++++++++ lib/sqlalchemy/sql/compiler.py | 9 ++++++-- lib/sqlalchemy/sql/default_comparator.py | 21 +++++++++++++---- lib/sqlalchemy/sql/elements.py | 2 +- lib/sqlalchemy/sql/operators.py | 5 ++++ lib/sqlalchemy/sql/sqltypes.py | 17 ++++++++++++++ lib/sqlalchemy/sql/type_api.py | 2 +- lib/sqlalchemy/types.py | 1 + test/dialect/mysql/test_query.py | 39 +++++++++++++++++++++++++++----- test/dialect/postgresql/test_query.py | 6 +++++ test/sql/test_operators.py | 28 ++++++++++++++++++++++- 14 files changed, 180 insertions(+), 16 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index ad9eefa09..f90ae40f8 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -1,3 +1,4 @@ + ============== 1.0 Changelog ============== @@ -21,6 +22,28 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, mysql + :tickets: 3263 + + The :meth:`.Operators.match` operator is now handled such that the + return type is not strictly assumed to be boolean; it now + returns a :class:`.Boolean` subclass called :class:`.MatchType`. + The type will still produce boolean behavior when used in Python + expressions, however the dialect can override its behavior at + result time. In the case of MySQL, while the MATCH operator + is typically used in a boolean context within an expression, + if one actually queries for the value of a match expression, a + floating point value is returned; this value is not compatible + with SQLAlchemy's C-based boolean processor, so MySQL's result-set + behavior now follows that of the :class:`.Float` type. + A new operator object ``notmatch_op`` is also added to better allow + dialects to define the negation of a match operation. + + .. seealso:: + + :ref:`change_3263` + .. change:: :tags: bug, postgresql :tickets: 3264 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index e148e7d70..929a5fe3d 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1547,6 +1547,37 @@ again works on MySQL. :ticket:`3186` +.. _change_3263: + +The match() operator now returns an agnostic MatchType compatible with MySQL's floating point return value +---------------------------------------------------------------------------------------------------------- + +The return type of a :meth:`.Operators.match` expression is now a new type +called :class:`.MatchType`. This is a subclass of :class:`.Boolean`, +that can be intercepted by the dialect in order to produce a different +result type at SQL execution time. + +Code like the following will now function correctly and return floating points +on MySQL:: + + >>> connection.execute( + ... select([ + ... matchtable.c.title.match('Agile Ruby Programming').label('ruby'), + ... matchtable.c.title.match('Dive Python').label('python'), + ... matchtable.c.title + ... ]).order_by(matchtable.c.id) + ... ) + [ + (2.0, 0.0, 'Agile Web Development with Ruby On Rails'), + (0.0, 2.0, 'Dive Into Python'), + (2.0, 0.0, "Programming Matz's Ruby"), + (0.0, 0.0, 'The Definitive Guide to Django'), + (0.0, 1.0, 'Python in a Nutshell') + ] + + +:ticket:`3263` + .. _change_3182: PyODBC driver name is required with hostname-based SQL Server connections diff --git a/doc/build/core/types.rst b/doc/build/core/types.rst index 14e30e46d..22b36a648 100644 --- a/doc/build/core/types.rst +++ b/doc/build/core/types.rst @@ -67,6 +67,9 @@ Standard Types`_ and the other sections of this chapter. .. autoclass:: LargeBinary :members: +.. autoclass:: MatchType + :members: + .. autoclass:: Numeric :members: diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 58eb3afa0..c868f58b2 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -602,6 +602,14 @@ class _StringType(sqltypes.String): to_inspect=[_StringType, sqltypes.String]) +class _MatchType(sqltypes.Float, sqltypes.MatchType): + def __init__(self, **kw): + # TODO: float arguments? + sqltypes.Float.__init__(self) + sqltypes.MatchType.__init__(self) + + + class NUMERIC(_NumericType, sqltypes.NUMERIC): """MySQL NUMERIC type.""" @@ -1544,6 +1552,7 @@ colspecs = { sqltypes.Float: FLOAT, sqltypes.Time: TIME, sqltypes.Enum: ENUM, + sqltypes.MatchType: _MatchType } # Everything 3.23 through 5.1 excepting OpenGIS types. diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index b102f0240..29a7401a1 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -82,6 +82,7 @@ OPERATORS = { operators.eq: ' = ', operators.concat_op: ' || ', operators.match_op: ' MATCH ', + operators.notmatch_op: ' NOT MATCH ', operators.in_op: ' IN ', operators.notin_op: ' NOT IN ', operators.comma_op: ', ', @@ -862,14 +863,18 @@ class SQLCompiler(Compiled): else: return "%s = 0" % self.process(element.element, **kw) - def visit_binary(self, binary, **kw): + def visit_notmatch_op_binary(self, binary, operator, **kw): + return "NOT %s" % self.visit_binary( + binary, override_operator=operators.match_op) + + def visit_binary(self, binary, override_operator=None, **kw): # don't allow "? = ?" to render if self.ansi_bind_rules and \ isinstance(binary.left, elements.BindParameter) and \ isinstance(binary.right, elements.BindParameter): kw['literal_binds'] = True - operator_ = binary.operator + operator_ = override_operator or binary.operator disp = getattr(self, "visit_%s_binary" % operator_.__name__, None) if disp: return disp(binary, operator_, **kw) diff --git a/lib/sqlalchemy/sql/default_comparator.py b/lib/sqlalchemy/sql/default_comparator.py index 4f53e2979..d26fdc455 100644 --- a/lib/sqlalchemy/sql/default_comparator.py +++ b/lib/sqlalchemy/sql/default_comparator.py @@ -68,8 +68,12 @@ class _DefaultColumnComparator(operators.ColumnOperators): def _boolean_compare(self, expr, op, obj, negate=None, reverse=False, _python_is_types=(util.NoneType, bool), + result_type = None, **kwargs): + if result_type is None: + result_type = type_api.BOOLEANTYPE + if isinstance(obj, _python_is_types + (Null, True_, False_)): # allow x ==/!= True/False to be treated as a literal. @@ -80,7 +84,7 @@ class _DefaultColumnComparator(operators.ColumnOperators): return BinaryExpression(expr, _literal_as_text(obj), op, - type_=type_api.BOOLEANTYPE, + type_=result_type, negate=negate, modifiers=kwargs) else: # all other None/True/False uses IS, IS NOT @@ -103,13 +107,13 @@ class _DefaultColumnComparator(operators.ColumnOperators): return BinaryExpression(obj, expr, op, - type_=type_api.BOOLEANTYPE, + type_=result_type, negate=negate, modifiers=kwargs) else: return BinaryExpression(expr, obj, op, - type_=type_api.BOOLEANTYPE, + type_=result_type, negate=negate, modifiers=kwargs) def _binary_operate(self, expr, op, obj, reverse=False, result_type=None, @@ -125,7 +129,8 @@ class _DefaultColumnComparator(operators.ColumnOperators): op, result_type = left.comparator._adapt_expression( op, right.comparator) - return BinaryExpression(left, right, op, type_=result_type) + return BinaryExpression( + left, right, op, type_=result_type, modifiers=kw) def _conjunction_operate(self, expr, op, other, **kw): if op is operators.and_: @@ -216,11 +221,16 @@ class _DefaultColumnComparator(operators.ColumnOperators): def _match_impl(self, expr, op, other, **kw): """See :meth:`.ColumnOperators.match`.""" + return self._boolean_compare( expr, operators.match_op, self._check_literal( expr, operators.match_op, other), - **kw) + result_type=type_api.MATCHTYPE, + negate=operators.notmatch_op + if op is operators.match_op else operators.match_op, + **kw + ) def _distinct_impl(self, expr, op, **kw): """See :meth:`.ColumnOperators.distinct`.""" @@ -282,6 +292,7 @@ class _DefaultColumnComparator(operators.ColumnOperators): "isnot": (_boolean_compare, operators.isnot), "collate": (_collate_impl,), "match_op": (_match_impl,), + "notmatch_op": (_match_impl,), "distinct_op": (_distinct_impl,), "between_op": (_between_impl, ), "notbetween_op": (_between_impl, ), diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index 734f78632..30965c801 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -2763,7 +2763,7 @@ class BinaryExpression(ColumnElement): self.right, self.negate, negate=self.operator, - type_=type_api.BOOLEANTYPE, + type_=self.type, modifiers=self.modifiers) else: return super(BinaryExpression, self)._negate() diff --git a/lib/sqlalchemy/sql/operators.py b/lib/sqlalchemy/sql/operators.py index 945356328..b08e44ab8 100644 --- a/lib/sqlalchemy/sql/operators.py +++ b/lib/sqlalchemy/sql/operators.py @@ -767,6 +767,10 @@ def match_op(a, b, **kw): return a.match(b, **kw) +def notmatch_op(a, b, **kw): + return a.notmatch(b, **kw) + + def comma_op(a, b): raise NotImplementedError() @@ -834,6 +838,7 @@ _PRECEDENCE = { concat_op: 6, match_op: 6, + notmatch_op: 6, ilike_op: 6, notilike_op: 6, diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 7bf2f337c..94db1d837 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -1654,10 +1654,26 @@ class NullType(TypeEngine): comparator_factory = Comparator +class MatchType(Boolean): + """Refers to the return type of the MATCH operator. + + As the :meth:`.Operators.match` is probably the most open-ended + operator in generic SQLAlchemy Core, we can't assume the return type + at SQL evaluation time, as MySQL returns a floating point, not a boolean, + and other backends might do something different. So this type + acts as a placeholder, currently subclassing :class:`.Boolean`. + The type allows dialects to inject result-processing functionality + if needed, and on MySQL will return floating-point values. + + .. versionadded:: 1.0.0 + + """ + NULLTYPE = NullType() BOOLEANTYPE = Boolean() STRINGTYPE = String() INTEGERTYPE = Integer() +MATCHTYPE = MatchType() _type_map = { int: Integer(), @@ -1685,6 +1701,7 @@ type_api.BOOLEANTYPE = BOOLEANTYPE type_api.STRINGTYPE = STRINGTYPE type_api.INTEGERTYPE = INTEGERTYPE type_api.NULLTYPE = NULLTYPE +type_api.MATCHTYPE = MATCHTYPE type_api._type_map = _type_map # this one, there's all kinds of ways to play it, but at the EOD diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py index 77c6e1b1e..d3e0a008e 100644 --- a/lib/sqlalchemy/sql/type_api.py +++ b/lib/sqlalchemy/sql/type_api.py @@ -19,7 +19,7 @@ BOOLEANTYPE = None INTEGERTYPE = None NULLTYPE = None STRINGTYPE = None - +MATCHTYPE = None class TypeEngine(Visitable): """The ultimate base class for all SQL datatypes. diff --git a/lib/sqlalchemy/types.py b/lib/sqlalchemy/types.py index b49e389ac..1215bd790 100644 --- a/lib/sqlalchemy/types.py +++ b/lib/sqlalchemy/types.py @@ -51,6 +51,7 @@ from .sql.sqltypes import ( Integer, Interval, LargeBinary, + MatchType, NCHAR, NVARCHAR, NullType, diff --git a/test/dialect/mysql/test_query.py b/test/dialect/mysql/test_query.py index e085d86c1..ccb501651 100644 --- a/test/dialect/mysql/test_query.py +++ b/test/dialect/mysql/test_query.py @@ -55,7 +55,7 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): ]) matchtable.insert().execute([ {'id': 1, - 'title': 'Agile Web Development with Rails', + 'title': 'Agile Web Development with Ruby On Rails', 'category_id': 2}, {'id': 2, 'title': 'Dive Into Python', @@ -76,7 +76,7 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): metadata.drop_all() @testing.fails_on('mysql+mysqlconnector', 'uses pyformat') - def test_expression(self): + def test_expression_format(self): format = testing.db.dialect.paramstyle == 'format' and '%s' or '?' self.assert_compile( matchtable.c.title.match('somstr'), @@ -88,7 +88,7 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): @testing.fails_on('mysql+oursql', 'uses format') @testing.fails_on('mysql+pyodbc', 'uses format') @testing.fails_on('mysql+zxjdbc', 'uses format') - def test_expression(self): + def test_expression_pyformat(self): format = '%(title_1)s' self.assert_compile( matchtable.c.title.match('somstr'), @@ -102,6 +102,14 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): fetchall()) eq_([2, 5], [r.id for r in results]) + def test_not_match(self): + results = (matchtable.select(). + where(~matchtable.c.title.match('python')). + order_by(matchtable.c.id). + execute(). + fetchall()) + eq_([1, 3, 4], [r.id for r in results]) + def test_simple_match_with_apostrophe(self): results = (matchtable.select(). where(matchtable.c.title.match("Matz's")). @@ -109,6 +117,26 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): fetchall()) eq_([3], [r.id for r in results]) + def test_return_value(self): + # test [ticket:3263] + result = testing.db.execute( + select([ + matchtable.c.title.match('Agile Ruby Programming').label('ruby'), + matchtable.c.title.match('Dive Python').label('python'), + matchtable.c.title + ]).order_by(matchtable.c.id) + ).fetchall() + eq_( + result, + [ + (2.0, 0.0, 'Agile Web Development with Ruby On Rails'), + (0.0, 2.0, 'Dive Into Python'), + (2.0, 0.0, "Programming Matz's Ruby"), + (0.0, 0.0, 'The Definitive Guide to Django'), + (0.0, 1.0, 'Python in a Nutshell') + ] + ) + def test_or_match(self): results1 = (matchtable.select(). where(or_(matchtable.c.title.match('nutshell'), @@ -116,14 +144,13 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): order_by(matchtable.c.id). execute(). fetchall()) - eq_([3, 5], [r.id for r in results1]) + eq_([1, 3, 5], [r.id for r in results1]) results2 = (matchtable.select(). where(matchtable.c.title.match('nutshell ruby')). order_by(matchtable.c.id). execute(). fetchall()) - eq_([3, 5], [r.id for r in results2]) - + eq_([1, 3, 5], [r.id for r in results2]) def test_and_match(self): results1 = (matchtable.select(). diff --git a/test/dialect/postgresql/test_query.py b/test/dialect/postgresql/test_query.py index a512b56fa..6841f397a 100644 --- a/test/dialect/postgresql/test_query.py +++ b/test/dialect/postgresql/test_query.py @@ -703,6 +703,12 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): matchtable.c.id).execute().fetchall() eq_([2, 5], [r.id for r in results]) + def test_not_match(self): + results = matchtable.select().where( + ~matchtable.c.title.match('python')).order_by( + matchtable.c.id).execute().fetchall() + eq_([1, 3, 4], [r.id for r in results]) + def test_simple_match_with_apostrophe(self): results = matchtable.select().where( matchtable.c.title.match("Matz's")).execute().fetchall() diff --git a/test/sql/test_operators.py b/test/sql/test_operators.py index e8ad88511..f8ac1528f 100644 --- a/test/sql/test_operators.py +++ b/test/sql/test_operators.py @@ -12,7 +12,8 @@ from sqlalchemy import exc from sqlalchemy.engine import default from sqlalchemy.sql.elements import _literal_as_text from sqlalchemy.schema import Column, Table, MetaData -from sqlalchemy.types import TypeEngine, TypeDecorator, UserDefinedType, Boolean +from sqlalchemy.types import TypeEngine, TypeDecorator, UserDefinedType, \ + Boolean, NullType, MatchType from sqlalchemy.dialects import mysql, firebird, postgresql, oracle, \ sqlite, mssql from sqlalchemy import util @@ -1619,6 +1620,31 @@ class MatchTest(fixtures.TestBase, testing.AssertsCompiledSQL): "CONTAINS (mytable.myid, :myid_1)", dialect=oracle.dialect()) + def test_match_is_now_matchtype(self): + expr = self.table1.c.myid.match('somstr') + assert expr.type._type_affinity is MatchType()._type_affinity + assert isinstance(expr.type, MatchType) + + def test_boolean_inversion_postgresql(self): + self.assert_compile( + ~self.table1.c.myid.match('somstr'), + "NOT mytable.myid @@ to_tsquery(%(myid_1)s)", + dialect=postgresql.dialect()) + + def test_boolean_inversion_mysql(self): + # because mysql doesnt have native boolean + self.assert_compile( + ~self.table1.c.myid.match('somstr'), + "NOT MATCH (mytable.myid) AGAINST (%s IN BOOLEAN MODE)", + dialect=mysql.dialect()) + + def test_boolean_inversion_mssql(self): + # because mssql doesnt have native boolean + self.assert_compile( + ~self.table1.c.myid.match('somstr'), + "NOT CONTAINS (mytable.myid, :myid_1)", + dialect=mssql.dialect()) + class ComposedLikeOperatorsTest(fixtures.TestBase, testing.AssertsCompiledSQL): __dialect__ = 'default' -- cgit v1.2.1 From fda589487b2cb60e8d69f520e0120eeb7c875915 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Dec 2014 19:12:52 -0500 Subject: - Updated the "supports_unicode_statements" flag to True for MySQLdb and Pymysql under Python 2. This refers to the SQL statements themselves, not the parameters, and affects issues such as table and column names using non-ASCII characters. These drivers both appear to support Python 2 Unicode objects without issue in modern versions. fixes #3121 --- doc/build/changelog/changelog_10.rst | 11 +++++++++++ lib/sqlalchemy/dialects/mysql/mysqldb.py | 2 +- lib/sqlalchemy/dialects/mysql/pymysql.py | 3 +-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index f90ae40f8..7126d0930 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,17 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, mysql + :tickets: 3121 + + Updated the "supports_unicode_statements" flag to True for MySQLdb + and Pymysql under Python 2. This refers to the SQL statements + themselves, not the parameters, and affects issues such as table + and column names using non-ASCII characters. These drivers both + appear to support Python 2 Unicode objects without issue in modern + versions. + .. change:: :tags: bug, mysql :tickets: 3263 diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index 73210d67a..893c6a9e2 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -77,7 +77,7 @@ class MySQLIdentifierPreparer_mysqldb(MySQLIdentifierPreparer): class MySQLDialect_mysqldb(MySQLDialect): driver = 'mysqldb' - supports_unicode_statements = False + supports_unicode_statements = True supports_sane_rowcount = True supports_sane_multi_rowcount = True diff --git a/lib/sqlalchemy/dialects/mysql/pymysql.py b/lib/sqlalchemy/dialects/mysql/pymysql.py index 31226cea0..8df2ba03f 100644 --- a/lib/sqlalchemy/dialects/mysql/pymysql.py +++ b/lib/sqlalchemy/dialects/mysql/pymysql.py @@ -31,8 +31,7 @@ class MySQLDialect_pymysql(MySQLDialect_mysqldb): driver = 'pymysql' description_encoding = None - if py3k: - supports_unicode_statements = True + supports_unicode_statements = True @classmethod def dbapi(cls): -- cgit v1.2.1 From e46c71b4198ee9811ea851dbe037f19a74af0b08 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Dec 2014 19:35:00 -0500 Subject: - Added support for CTEs under Oracle. This includes some tweaks to the aliasing syntax, as well as a new CTE feature :meth:`.CTE.suffix_with`, which is useful for adding in special Oracle-specific directives to the CTE. fixes #3220 --- doc/build/changelog/changelog_10.rst | 13 ++++ doc/build/changelog/migration_10.rst | 18 +++++ doc/build/core/selectable.rst | 3 + lib/sqlalchemy/dialects/oracle/base.py | 21 ++---- lib/sqlalchemy/orm/query.py | 30 +++++++- lib/sqlalchemy/sql/compiler.py | 17 ++++- lib/sqlalchemy/sql/selectable.py | 131 ++++++++++++++++++++++----------- test/dialect/test_oracle.py | 45 +++++++++++ test/sql/test_cte.py | 30 ++++++++ 9 files changed, 247 insertions(+), 61 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 7126d0930..32fe4daab 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,19 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, oracle + :tickets: 3220 + + Added support for CTEs under Oracle. This includes some tweaks + to the aliasing syntax, as well as a new CTE feature + :meth:`.CTE.suffix_with`, which is useful for adding in special + Oracle-specific directives to the CTE. + + .. seealso:: + + :ref:`change_3220` + .. change:: :tags: feature, mysql :tickets: 3121 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 929a5fe3d..9fbbb889d 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1616,6 +1616,24 @@ reflection from temp tables as well, which is :ticket:`3203`. :ticket:`3204` +.. _change_3220: + +Improved support for CTEs in Oracle +----------------------------------- + +CTE support has been fixed up for Oracle, and there is also a new feature +:meth:`.CTE.with_suffixes` that can assist with Oracle's special directives:: + + included_parts = select([ + part.c.sub_part, part.c.part, part.c.quantity + ]).where(part.c.part == "p1").\ + cte(name="included_parts", recursive=True).\ + suffix_with( + "search depth first by part set ord1", + "cycle part set y_cycle to 1 default 0", dialect='oracle') + +:ticket:`3220` + .. _change_2984: Drizzle Dialect is now an External Dialect diff --git a/doc/build/core/selectable.rst b/doc/build/core/selectable.rst index 52acb28e5..03ebeb4ab 100644 --- a/doc/build/core/selectable.rst +++ b/doc/build/core/selectable.rst @@ -60,6 +60,9 @@ elements are themselves :class:`.ColumnElement` subclasses). .. autoclass:: HasPrefixes :members: +.. autoclass:: HasSuffixes + :members: + .. autoclass:: Join :members: :inherited-members: diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py index 6df38e57e..524ba8115 100644 --- a/lib/sqlalchemy/dialects/oracle/base.py +++ b/lib/sqlalchemy/dialects/oracle/base.py @@ -549,6 +549,9 @@ class OracleCompiler(compiler.SQLCompiler): def visit_false(self, expr, **kw): return '0' + def get_cte_preamble(self, recursive): + return "WITH" + def get_select_hint_text(self, byfroms): return " ".join( "/*+ %s */" % text for table, text in byfroms.items() @@ -619,22 +622,10 @@ class OracleCompiler(compiler.SQLCompiler): return (self.dialect.identifier_preparer.format_sequence(seq) + ".nextval") - def visit_alias(self, alias, asfrom=False, ashint=False, **kwargs): - """Oracle doesn't like ``FROM table AS alias``. Is the AS standard - SQL?? - """ - - if asfrom or ashint: - alias_name = isinstance(alias.name, expression._truncated_label) and \ - self._truncated_identifier("alias", alias.name) or alias.name + def get_render_as_alias_suffix(self, alias_name_text): + """Oracle doesn't like ``FROM table AS alias``""" - if ashint: - return alias_name - elif asfrom: - return self.process(alias.original, asfrom=asfrom, **kwargs) + \ - " " + self.preparer.format_alias(alias, alias_name) - else: - return self.process(alias.original, **kwargs) + return " " + alias_name_text def returning_clause(self, stmt, returning_cols): columns = [] diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 790686288..9b7747e15 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -75,6 +75,7 @@ class Query(object): _having = None _distinct = False _prefixes = None + _suffixes = None _offset = None _limit = None _for_update_arg = None @@ -1003,7 +1004,7 @@ class Query(object): '_limit', '_offset', '_joinpath', '_joinpoint', '_distinct', '_having', - '_prefixes', + '_prefixes', '_suffixes' ): self.__dict__.pop(attr, None) self._set_select_from([fromclause], True) @@ -2359,12 +2360,38 @@ class Query(object): .. versionadded:: 0.7.7 + .. seealso:: + + :meth:`.HasPrefixes.prefix_with` + """ if self._prefixes: self._prefixes += prefixes else: self._prefixes = prefixes + @_generative() + def suffix_with(self, *suffixes): + """Apply the suffix to the query and return the newly resulting + ``Query``. + + :param \*suffixes: optional suffixes, typically strings, + not using any commas. + + .. versionadded:: 1.0.0 + + .. seealso:: + + :meth:`.Query.prefix_with` + + :meth:`.HasSuffixes.suffix_with` + + """ + if self._suffixes: + self._suffixes += suffixes + else: + self._suffixes = suffixes + def all(self): """Return the results represented by this ``Query`` as a list. @@ -2601,6 +2628,7 @@ class Query(object): 'offset': self._offset, 'distinct': self._distinct, 'prefixes': self._prefixes, + 'suffixes': self._suffixes, 'group_by': self._group_by or None, 'having': self._having } diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 29a7401a1..9304bba9f 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -1193,12 +1193,16 @@ class SQLCompiler(Compiled): self, asfrom=True, **kwargs ) + if cte._suffixes: + text += " " + self._generate_prefixes( + cte, cte._suffixes, **kwargs) + self.ctes[cte] = text if asfrom: if cte_alias_name: text = self.preparer.format_alias(cte, cte_alias_name) - text += " AS " + cte_name + text += self.get_render_as_alias_suffix(cte_name) else: return self.preparer.format_alias(cte, cte_name) return text @@ -1217,8 +1221,8 @@ class SQLCompiler(Compiled): elif asfrom: ret = alias.original._compiler_dispatch(self, asfrom=True, **kwargs) + \ - " AS " + \ - self.preparer.format_alias(alias, alias_name) + self.get_render_as_alias_suffix( + self.preparer.format_alias(alias, alias_name)) if fromhints and alias in fromhints: ret = self.format_from_hint_text(ret, alias, @@ -1228,6 +1232,9 @@ class SQLCompiler(Compiled): else: return alias.original._compiler_dispatch(self, **kwargs) + def get_render_as_alias_suffix(self, alias_name_text): + return " AS " + alias_name_text + def _add_to_result_map(self, keyname, name, objects, type_): if not self.dialect.case_sensitive: keyname = keyname.lower() @@ -1554,6 +1561,10 @@ class SQLCompiler(Compiled): compound_index == 0 and toplevel: text = self._render_cte_clause() + text + if select._suffixes: + text += " " + self._generate_prefixes( + select, select._suffixes, **kwargs) + self.stack.pop(-1) if asfrom and parens: diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py index 8198a6733..87029ec2b 100644 --- a/lib/sqlalchemy/sql/selectable.py +++ b/lib/sqlalchemy/sql/selectable.py @@ -171,6 +171,79 @@ class Selectable(ClauseElement): return self +class HasPrefixes(object): + _prefixes = () + + @_generative + def prefix_with(self, *expr, **kw): + """Add one or more expressions following the statement keyword, i.e. + SELECT, INSERT, UPDATE, or DELETE. Generative. + + This is used to support backend-specific prefix keywords such as those + provided by MySQL. + + E.g.:: + + stmt = table.insert().prefix_with("LOW_PRIORITY", dialect="mysql") + + Multiple prefixes can be specified by multiple calls + to :meth:`.prefix_with`. + + :param \*expr: textual or :class:`.ClauseElement` construct which + will be rendered following the INSERT, UPDATE, or DELETE + keyword. + :param \**kw: A single keyword 'dialect' is accepted. This is an + optional string dialect name which will + limit rendering of this prefix to only that dialect. + + """ + dialect = kw.pop('dialect', None) + if kw: + raise exc.ArgumentError("Unsupported argument(s): %s" % + ",".join(kw)) + self._setup_prefixes(expr, dialect) + + def _setup_prefixes(self, prefixes, dialect=None): + self._prefixes = self._prefixes + tuple( + [(_literal_as_text(p, warn=False), dialect) for p in prefixes]) + + +class HasSuffixes(object): + _suffixes = () + + @_generative + def suffix_with(self, *expr, **kw): + """Add one or more expressions following the statement as a whole. + + This is used to support backend-specific suffix keywords on + certain constructs. + + E.g.:: + + stmt = select([col1, col2]).cte().suffix_with( + "cycle empno set y_cycle to 1 default 0", dialect="oracle") + + Multiple prefixes can be specified by multiple calls + to :meth:`.suffix_with`. + + :param \*expr: textual or :class:`.ClauseElement` construct which + will be rendered following the target clause. + :param \**kw: A single keyword 'dialect' is accepted. This is an + optional string dialect name which will + limit rendering of this suffix to only that dialect. + + """ + dialect = kw.pop('dialect', None) + if kw: + raise exc.ArgumentError("Unsupported argument(s): %s" % + ",".join(kw)) + self._setup_suffixes(expr, dialect) + + def _setup_suffixes(self, suffixes, dialect=None): + self._suffixes = self._suffixes + tuple( + [(_literal_as_text(p, warn=False), dialect) for p in suffixes]) + + class FromClause(Selectable): """Represent an element that can be used within the ``FROM`` clause of a ``SELECT`` statement. @@ -1088,7 +1161,7 @@ class Alias(FromClause): return self.element.bind -class CTE(Alias): +class CTE(Generative, HasSuffixes, Alias): """Represent a Common Table Expression. The :class:`.CTE` object is obtained using the @@ -1104,10 +1177,13 @@ class CTE(Alias): name=None, recursive=False, _cte_alias=None, - _restates=frozenset()): + _restates=frozenset(), + _suffixes=None): self.recursive = recursive self._cte_alias = _cte_alias self._restates = _restates + if _suffixes: + self._suffixes = _suffixes super(CTE, self).__init__(selectable, name=name) def alias(self, name=None, flat=False): @@ -1116,6 +1192,7 @@ class CTE(Alias): name=name, recursive=self.recursive, _cte_alias=self, + _suffixes=self._suffixes ) def union(self, other): @@ -1123,7 +1200,8 @@ class CTE(Alias): self.original.union(other), name=self.name, recursive=self.recursive, - _restates=self._restates.union([self]) + _restates=self._restates.union([self]), + _suffixes=self._suffixes ) def union_all(self, other): @@ -1131,7 +1209,8 @@ class CTE(Alias): self.original.union_all(other), name=self.name, recursive=self.recursive, - _restates=self._restates.union([self]) + _restates=self._restates.union([self]), + _suffixes=self._suffixes ) @@ -2118,44 +2197,7 @@ class CompoundSelect(GenerativeSelect): bind = property(bind, _set_bind) -class HasPrefixes(object): - _prefixes = () - - @_generative - def prefix_with(self, *expr, **kw): - """Add one or more expressions following the statement keyword, i.e. - SELECT, INSERT, UPDATE, or DELETE. Generative. - - This is used to support backend-specific prefix keywords such as those - provided by MySQL. - - E.g.:: - - stmt = table.insert().prefix_with("LOW_PRIORITY", dialect="mysql") - - Multiple prefixes can be specified by multiple calls - to :meth:`.prefix_with`. - - :param \*expr: textual or :class:`.ClauseElement` construct which - will be rendered following the INSERT, UPDATE, or DELETE - keyword. - :param \**kw: A single keyword 'dialect' is accepted. This is an - optional string dialect name which will - limit rendering of this prefix to only that dialect. - - """ - dialect = kw.pop('dialect', None) - if kw: - raise exc.ArgumentError("Unsupported argument(s): %s" % - ",".join(kw)) - self._setup_prefixes(expr, dialect) - - def _setup_prefixes(self, prefixes, dialect=None): - self._prefixes = self._prefixes + tuple( - [(_literal_as_text(p, warn=False), dialect) for p in prefixes]) - - -class Select(HasPrefixes, GenerativeSelect): +class Select(HasPrefixes, HasSuffixes, GenerativeSelect): """Represents a ``SELECT`` statement. """ @@ -2163,6 +2205,7 @@ class Select(HasPrefixes, GenerativeSelect): __visit_name__ = 'select' _prefixes = () + _suffixes = () _hints = util.immutabledict() _statement_hints = () _distinct = False @@ -2180,6 +2223,7 @@ class Select(HasPrefixes, GenerativeSelect): having=None, correlate=True, prefixes=None, + suffixes=None, **kwargs): """Construct a new :class:`.Select`. @@ -2425,6 +2469,9 @@ class Select(HasPrefixes, GenerativeSelect): if prefixes: self._setup_prefixes(prefixes) + if suffixes: + self._setup_suffixes(suffixes) + GenerativeSelect.__init__(self, **kwargs) @property diff --git a/test/dialect/test_oracle.py b/test/dialect/test_oracle.py index a771c5d80..b2a490e71 100644 --- a/test/dialect/test_oracle.py +++ b/test/dialect/test_oracle.py @@ -180,6 +180,51 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): t.update().values(plain=5), 'UPDATE s SET "plain"=:"plain"' ) + def test_cte(self): + part = table( + 'part', + column('part'), + column('sub_part'), + column('quantity') + ) + + included_parts = select([ + part.c.sub_part, part.c.part, part.c.quantity + ]).where(part.c.part == "p1").\ + cte(name="included_parts", recursive=True).\ + suffix_with( + "search depth first by part set ord1", + "cycle part set y_cycle to 1 default 0", dialect='oracle') + + incl_alias = included_parts.alias("pr1") + parts_alias = part.alias("p") + included_parts = included_parts.union_all( + select([ + parts_alias.c.sub_part, + parts_alias.c.part, parts_alias.c.quantity + ]).where(parts_alias.c.part == incl_alias.c.sub_part) + ) + + q = select([ + included_parts.c.sub_part, + func.sum(included_parts.c.quantity).label('total_quantity')]).\ + group_by(included_parts.c.sub_part) + + self.assert_compile( + q, + "WITH included_parts(sub_part, part, quantity) AS " + "(SELECT part.sub_part AS sub_part, part.part AS part, " + "part.quantity AS quantity FROM part WHERE part.part = :part_1 " + "UNION ALL SELECT p.sub_part AS sub_part, p.part AS part, " + "p.quantity AS quantity FROM part p, included_parts pr1 " + "WHERE p.part = pr1.sub_part) " + "search depth first by part set ord1 cycle part set " + "y_cycle to 1 default 0 " + "SELECT included_parts.sub_part, sum(included_parts.quantity) " + "AS total_quantity FROM included_parts " + "GROUP BY included_parts.sub_part" + ) + def test_limit(self): t = table('sometable', column('col1'), column('col2')) s = select([t]) diff --git a/test/sql/test_cte.py b/test/sql/test_cte.py index b907fe649..c7906dcb7 100644 --- a/test/sql/test_cte.py +++ b/test/sql/test_cte.py @@ -462,3 +462,33 @@ class CTETest(fixtures.TestBase, AssertsCompiledSQL): 'FROM "order" JOIN regional_sales AS anon_1 ' 'ON anon_1."order" = "order"."order"' ) + + def test_suffixes(self): + orders = table('order', column('order')) + s = select([orders.c.order]).cte("regional_sales") + s = s.suffix_with("pg suffix", dialect='postgresql') + s = s.suffix_with('oracle suffix', dialect='oracle') + stmt = select([orders]).where(orders.c.order > s.c.order) + + self.assert_compile( + stmt, + 'WITH regional_sales AS (SELECT "order"."order" AS "order" ' + 'FROM "order") SELECT "order"."order" FROM "order", ' + 'regional_sales WHERE "order"."order" > regional_sales."order"' + ) + + self.assert_compile( + stmt, + 'WITH regional_sales AS (SELECT "order"."order" AS "order" ' + 'FROM "order") oracle suffix SELECT "order"."order" FROM "order", ' + 'regional_sales WHERE "order"."order" > regional_sales."order"', + dialect='oracle' + ) + + self.assert_compile( + stmt, + 'WITH regional_sales AS (SELECT "order"."order" AS "order" ' + 'FROM "order") pg suffix SELECT "order"."order" FROM "order", ' + 'regional_sales WHERE "order"."order" > regional_sales."order"', + dialect='postgresql' + ) \ No newline at end of file -- cgit v1.2.1 From 60174146410d4ce2a17faa76cd981f558490db92 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Dec 2014 19:45:14 -0500 Subject: - the refactor of the visit_alias() method in Oracle revealed that quoting should be applied in %(name)s under with_hint. --- doc/build/changelog/changelog_10.rst | 7 +++++++ test/sql/test_compiler.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 32fe4daab..0256958b2 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,13 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, oracle + + An alias name will be properly quoted when referred to using the + ``%(name)s`` token inside the :meth:`.Select.with_hint` method. + Previously, the Oracle backend hadn't implemented this quoting. + .. change:: :tags: feature, oracle :tickets: 3220 diff --git a/test/sql/test_compiler.py b/test/sql/test_compiler.py index 9e99a947b..428fc8986 100644 --- a/test/sql/test_compiler.py +++ b/test/sql/test_compiler.py @@ -2440,7 +2440,7 @@ class SelectTest(fixtures.TestBase, AssertsCompiledSQL): """SELECT /*+ "QuotedName" idx1 */ "QuotedName".col1 """ """FROM "QuotedName" WHERE "QuotedName".col1 > :col1_1"""), (s7, oracle_d, - """SELECT /*+ SomeName idx1 */ "SomeName".col1 FROM """ + """SELECT /*+ "SomeName" idx1 */ "SomeName".col1 FROM """ """"QuotedName" "SomeName" WHERE "SomeName".col1 > :col1_1"""), ]: self.assert_compile( -- cgit v1.2.1 From edef95379777a9c84ee7dbcbc9a3b58849aa8930 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 4 Dec 2014 20:08:07 -0500 Subject: - New Oracle DDL features for tables, indexes: COMPRESS, BITMAP. Patch courtesy Gabor Gombas. fixes #3127 --- doc/build/changelog/changelog_10.rst | 6 ++ doc/build/changelog/migration_10.rst | 9 ++ lib/sqlalchemy/dialects/oracle/base.py | 165 +++++++++++++++++++++++++++++++-- lib/sqlalchemy/engine/reflection.py | 10 +- test/dialect/test_oracle.py | 93 ++++++++++++++++++- 5 files changed, 272 insertions(+), 11 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 0256958b2..b71ecc15d 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,12 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, oracle + + New Oracle DDL features for tables, indexes: COMPRESS, BITMAP. + Patch courtesy Gabor Gombas. + .. change:: :tags: bug, oracle diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 9fbbb889d..27a4fae4c 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1634,6 +1634,15 @@ CTE support has been fixed up for Oracle, and there is also a new feature :ticket:`3220` +New Oracle Keywords for DDL +----------------------------- + +Keywords such as COMPRESS, ON COMMIT, BITMAP: + +:ref:`oracle_table_options` + +:ref:`oracle_index_options` + .. _change_2984: Drizzle Dialect is now an External Dialect diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py index 524ba8115..9f375da94 100644 --- a/lib/sqlalchemy/dialects/oracle/base.py +++ b/lib/sqlalchemy/dialects/oracle/base.py @@ -213,6 +213,8 @@ is reflected and the type is reported as ``DATE``, the time-supporting examining the type of column for use in special Python translations or for migrating schemas to other database backends. +.. _oracle_table_options: + Oracle Table Options ------------------------- @@ -228,15 +230,63 @@ in conjunction with the :class:`.Table` construct: .. versionadded:: 1.0.0 +* ``COMPRESS``:: + + Table('mytable', metadata, Column('data', String(32)), + oracle_compress=True) + + Table('mytable', metadata, Column('data', String(32)), + oracle_compress=6) + + The ``oracle_compress`` parameter accepts either an integer compression + level, or ``True`` to use the default compression level. + +.. versionadded:: 1.0.0 + +.. _oracle_index_options: + +Oracle Specific Index Options +----------------------------- + +Bitmap Indexes +~~~~~~~~~~~~~~ + +You can specify the ``oracle_bitmap`` parameter to create a bitmap index +instead of a B-tree index:: + + Index('my_index', my_table.c.data, oracle_bitmap=True) + +Bitmap indexes cannot be unique and cannot be compressed. SQLAlchemy will not +check for such limitations, only the database will. + +.. versionadded:: 1.0.0 + +Index compression +~~~~~~~~~~~~~~~~~ + +Oracle has a more efficient storage mode for indexes containing lots of +repeated values. Use the ``oracle_compress`` parameter to turn on key c +ompression:: + + Index('my_index', my_table.c.data, oracle_compress=True) + + Index('my_index', my_table.c.data1, my_table.c.data2, unique=True, + oracle_compress=1) + +The ``oracle_compress`` parameter accepts either an integer specifying the +number of prefix columns to compress, or ``True`` to use the default (all +columns for non-unique indexes, all but the last column for unique indexes). + +.. versionadded:: 1.0.0 + """ import re from sqlalchemy import util, sql -from sqlalchemy.engine import default, base, reflection +from sqlalchemy.engine import default, reflection from sqlalchemy.sql import compiler, visitors, expression -from sqlalchemy.sql import (operators as sql_operators, - functions as sql_functions) +from sqlalchemy.sql import operators as sql_operators from sqlalchemy import types as sqltypes, schema as sa_schema from sqlalchemy.types import VARCHAR, NVARCHAR, CHAR, \ BLOB, CLOB, TIMESTAMP, FLOAT @@ -786,9 +836,32 @@ class OracleDDLCompiler(compiler.DDLCompiler): return text - def visit_create_index(self, create, **kw): - return super(OracleDDLCompiler, self).\ - visit_create_index(create, include_schema=True) + def visit_create_index(self, create): + index = create.element + self._verify_index_table(index) + preparer = self.preparer + text = "CREATE " + if index.unique: + text += "UNIQUE " + if index.dialect_options['oracle']['bitmap']: + text += "BITMAP " + text += "INDEX %s ON %s (%s)" % ( + self._prepared_index_name(index, include_schema=True), + preparer.format_table(index.table, use_schema=True), + ', '.join( + self.sql_compiler.process( + expr, + include_table=False, literal_binds=True) + for expr in index.expressions) + ) + if index.dialect_options['oracle']['compress'] is not False: + if index.dialect_options['oracle']['compress'] is True: + text += " COMPRESS" + else: + text += " COMPRESS %d" % ( + index.dialect_options['oracle']['compress'] + ) + return text def post_create_table(self, table): table_opts = [] @@ -798,6 +871,14 @@ class OracleDDLCompiler(compiler.DDLCompiler): on_commit_options = opts['on_commit'].replace("_", " ").upper() table_opts.append('\n ON COMMIT %s' % on_commit_options) + if opts['compress']: + if opts['compress'] is True: + table_opts.append("\n COMPRESS") + else: + table_opts.append("\n COMPRESS FOR %s" % ( + opts['compress'] + )) + return ''.join(table_opts) @@ -861,7 +942,12 @@ class OracleDialect(default.DefaultDialect): construct_arguments = [ (sa_schema.Table, { "resolve_synonyms": False, - "on_commit": None + "on_commit": None, + "compress": False + }), + (sa_schema.Index, { + "bitmap": False, + "compress": False }) ] @@ -892,6 +978,16 @@ class OracleDialect(default.DefaultDialect): return self.server_version_info and \ self.server_version_info < (9, ) + @property + def _supports_table_compression(self): + return self.server_version_info and \ + self.server_version_info >= (9, 2, ) + + @property + def _supports_table_compress_for(self): + return self.server_version_info and \ + self.server_version_info >= (11, ) + @property def _supports_char_length(self): return not self._is_oracle_8 @@ -1074,6 +1170,50 @@ class OracleDialect(default.DefaultDialect): cursor = connection.execute(s, owner=self.denormalize_name(schema)) return [self.normalize_name(row[0]) for row in cursor] + @reflection.cache + def get_table_options(self, connection, table_name, schema=None, **kw): + options = {} + + resolve_synonyms = kw.get('oracle_resolve_synonyms', False) + dblink = kw.get('dblink', '') + info_cache = kw.get('info_cache') + + (table_name, schema, dblink, synonym) = \ + self._prepare_reflection_args(connection, table_name, schema, + resolve_synonyms, dblink, + info_cache=info_cache) + + params = {"table_name": table_name} + + columns = ["table_name"] + if self._supports_table_compression: + columns.append("compression") + if self._supports_table_compress_for: + columns.append("compress_for") + + text = "SELECT %(columns)s "\ + "FROM ALL_TABLES%(dblink)s "\ + "WHERE table_name = :table_name" + + if schema is not None: + params['owner'] = schema + text += " AND owner = :owner " + text = text % {'dblink': dblink, 'columns': ", ".join(columns)} + + result = connection.execute(sql.text(text), **params) + + enabled = dict(DISABLED=False, ENABLED=True) + + row = result.first() + if row: + if "compression" in row and enabled.get(row.compression, False): + if "compress_for" in row: + options['oracle_compress'] = row.compress_for + else: + options['oracle_compress'] = True + + return options + @reflection.cache def get_columns(self, connection, table_name, schema=None, **kw): """ @@ -1159,7 +1299,8 @@ class OracleDialect(default.DefaultDialect): params = {'table_name': table_name} text = \ - "SELECT a.index_name, a.column_name, b.uniqueness "\ + "SELECT a.index_name, a.column_name, "\ + "\nb.index_type, b.uniqueness, b.compression, b.prefix_length "\ "\nFROM ALL_IND_COLUMNS%(dblink)s a, "\ "\nALL_INDEXES%(dblink)s b "\ "\nWHERE "\ @@ -1185,6 +1326,7 @@ class OracleDialect(default.DefaultDialect): dblink=dblink, info_cache=kw.get('info_cache')) pkeys = pk_constraint['constrained_columns'] uniqueness = dict(NONUNIQUE=False, UNIQUE=True) + enabled = dict(DISABLED=False, ENABLED=True) oracle_sys_col = re.compile(r'SYS_NC\d+\$', re.IGNORECASE) @@ -1204,10 +1346,15 @@ class OracleDialect(default.DefaultDialect): if rset.index_name != last_index_name: remove_if_primary_key(index) index = dict(name=self.normalize_name(rset.index_name), - column_names=[]) + column_names=[], dialect_options={}) indexes.append(index) index['unique'] = uniqueness.get(rset.uniqueness, False) + if rset.index_type in ('BITMAP', 'FUNCTION-BASED BITMAP'): + index['dialect_options']['oracle_bitmap'] = True + if enabled.get(rset.compression, False): + index['dialect_options']['oracle_compress'] = rset.prefix_length + # filter out Oracle SYS_NC names. could also do an outer join # to the all_tab_columns table and check for real col names there. if not oracle_sys_col.match(rset.column_name): diff --git a/lib/sqlalchemy/engine/reflection.py b/lib/sqlalchemy/engine/reflection.py index 2a1def86a..ebc96f5dd 100644 --- a/lib/sqlalchemy/engine/reflection.py +++ b/lib/sqlalchemy/engine/reflection.py @@ -394,6 +394,9 @@ class Inspector(object): unique boolean + dialect_options + dict of dialect-specific index options + :param table_name: string name of the table. For special quoting, use :class:`.quoted_name`. @@ -642,6 +645,8 @@ class Inspector(object): columns = index_d['column_names'] unique = index_d['unique'] flavor = index_d.get('type', 'index') + dialect_options = index_d.get('dialect_options', {}) + duplicates = index_d.get('duplicates_constraint') if include_columns and \ not set(columns).issubset(include_columns): @@ -667,7 +672,10 @@ class Inspector(object): else: idx_cols.append(idx_col) - sa_schema.Index(name, *idx_cols, **dict(unique=unique)) + sa_schema.Index( + name, *idx_cols, + **dict(list(dialect_options.items()) + [('unique', unique)]) + ) def _reflect_unique_constraints( self, table_name, schema, table, cols_by_orig_name, diff --git a/test/dialect/test_oracle.py b/test/dialect/test_oracle.py index b2a490e71..1e50b9070 100644 --- a/test/dialect/test_oracle.py +++ b/test/dialect/test_oracle.py @@ -732,6 +732,34 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): ) + def test_create_table_compress(self): + m = MetaData() + tbl1 = Table('testtbl1', m, Column('data', Integer), + oracle_compress=True) + tbl2 = Table('testtbl2', m, Column('data', Integer), + oracle_compress="OLTP") + + self.assert_compile(schema.CreateTable(tbl1), + "CREATE TABLE testtbl1 (data INTEGER) COMPRESS") + self.assert_compile(schema.CreateTable(tbl2), + "CREATE TABLE testtbl2 (data INTEGER) " + "COMPRESS FOR OLTP") + + def test_create_index_bitmap_compress(self): + m = MetaData() + tbl = Table('testtbl', m, Column('data', Integer)) + idx1 = Index('idx1', tbl.c.data, oracle_compress=True) + idx2 = Index('idx2', tbl.c.data, oracle_compress=1) + idx3 = Index('idx3', tbl.c.data, oracle_bitmap=True) + + self.assert_compile(schema.CreateIndex(idx1), + "CREATE INDEX idx1 ON testtbl (data) COMPRESS") + self.assert_compile(schema.CreateIndex(idx2), + "CREATE INDEX idx2 ON testtbl (data) COMPRESS 1") + self.assert_compile(schema.CreateIndex(idx3), + "CREATE BITMAP INDEX idx3 ON testtbl (data)") + + class CompatFlagsTest(fixtures.TestBase, AssertsCompiledSQL): def _dialect(self, server_version, **kw): @@ -1772,6 +1800,58 @@ class UnsupportedIndexReflectTest(fixtures.TestBase): m2 = MetaData(testing.db) Table('test_index_reflect', m2, autoload=True) + +def all_tables_compression_missing(): + try: + testing.db.execute('SELECT compression FROM all_tables') + return False + except: + return True + + +def all_tables_compress_for_missing(): + try: + testing.db.execute('SELECT compress_for FROM all_tables') + return False + except: + return True + + +class TableReflectionTest(fixtures.TestBase): + __only_on__ = 'oracle' + + @testing.provide_metadata + @testing.fails_if(all_tables_compression_missing) + def test_reflect_basic_compression(self): + metadata = self.metadata + + tbl = Table('test_compress', metadata, + Column('data', Integer, primary_key=True), + oracle_compress=True) + metadata.create_all() + + m2 = MetaData(testing.db) + + tbl = Table('test_compress', m2, autoload=True) + # Don't hardcode the exact value, but it must be non-empty + assert tbl.dialect_options['oracle']['compress'] + + @testing.provide_metadata + @testing.fails_if(all_tables_compress_for_missing) + def test_reflect_oltp_compression(self): + metadata = self.metadata + + tbl = Table('test_compress', metadata, + Column('data', Integer, primary_key=True), + oracle_compress="OLTP") + metadata.create_all() + + m2 = MetaData(testing.db) + + tbl = Table('test_compress', m2, autoload=True) + assert tbl.dialect_options['oracle']['compress'] == "OLTP" + + class RoundTripIndexTest(fixtures.TestBase): __only_on__ = 'oracle' @@ -1789,6 +1869,10 @@ class RoundTripIndexTest(fixtures.TestBase): # "group" is a keyword, so lower case normalind = Index('tableind', table.c.id_b, table.c.group) + compress1 = Index('compress1', table.c.id_a, table.c.id_b, + oracle_compress=True) + compress2 = Index('compress2', table.c.id_a, table.c.id_b, table.c.col, + oracle_compress=1) metadata.create_all() mirror = MetaData(testing.db) @@ -1837,8 +1921,15 @@ class RoundTripIndexTest(fixtures.TestBase): ) assert (Index, ('id_b', ), True) in reflected assert (Index, ('col', 'group'), True) in reflected + + idx = reflected[(Index, ('id_a', 'id_b', ), False)] + assert idx.dialect_options['oracle']['compress'] == 2 + + idx = reflected[(Index, ('id_a', 'id_b', 'col', ), False)] + assert idx.dialect_options['oracle']['compress'] == 1 + eq_(len(reflectedtable.constraints), 1) - eq_(len(reflectedtable.indexes), 3) + eq_(len(reflectedtable.indexes), 5) class SequenceTest(fixtures.TestBase, AssertsCompiledSQL): -- cgit v1.2.1 From 6e53e866dea4eba630128e856573ca1076b91611 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 11:35:42 -0500 Subject: - pep8 cleanup --- test/engine/test_parseconnect.py | 135 ++++++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 58 deletions(-) diff --git a/test/engine/test_parseconnect.py b/test/engine/test_parseconnect.py index 391b92144..d8f202f99 100644 --- a/test/engine/test_parseconnect.py +++ b/test/engine/test_parseconnect.py @@ -1,12 +1,13 @@ from sqlalchemy.testing import assert_raises, eq_, assert_raises_message -from sqlalchemy.util.compat import configparser, StringIO import sqlalchemy.engine.url as url from sqlalchemy import create_engine, engine_from_config, exc, pool from sqlalchemy.engine.default import DefaultDialect import sqlalchemy as tsa from sqlalchemy.testing import fixtures from sqlalchemy import testing -from sqlalchemy.testing.mock import Mock, MagicMock, patch +from sqlalchemy.testing.mock import Mock, MagicMock + +dialect = None class ParseConnectTest(fixtures.TestBase): @@ -31,21 +32,25 @@ class ParseConnectTest(fixtures.TestBase): 'dbtype://username:password@/database', 'dbtype:////usr/local/_xtest@example.com/members.db', 'dbtype://username:apples%2Foranges@hostspec/database', - 'dbtype://username:password@[2001:da8:2004:1000:202:116:160:90]/database?foo=bar', - 'dbtype://username:password@[2001:da8:2004:1000:202:116:160:90]:80/database?foo=bar' - ): + 'dbtype://username:password@[2001:da8:2004:1000:202:116:160:90]' + '/database?foo=bar', + 'dbtype://username:password@[2001:da8:2004:1000:202:116:160:90]:80' + '/database?foo=bar' + ): u = url.make_url(text) assert u.drivername in ('dbtype', 'dbtype+apitype') assert u.username in ('username', None) assert u.password in ('password', 'apples/oranges', None) - assert u.host in ('hostspec', '127.0.0.1', - '2001:da8:2004:1000:202:116:160:90', '', None), u.host - assert u.database in ('database', - '/usr/local/_xtest@example.com/members.db', - '/usr/db_file.db', ':memory:', '', - 'foo/bar/im/a/file', - 'E:/work/src/LEM/db/hello.db', None), u.database + assert u.host in ( + 'hostspec', '127.0.0.1', + '2001:da8:2004:1000:202:116:160:90', '', None), u.host + assert u.database in ( + 'database', + '/usr/local/_xtest@example.com/members.db', + '/usr/db_file.db', ':memory:', '', + 'foo/bar/im/a/file', + 'E:/work/src/LEM/db/hello.db', None), u.database eq_(str(u), text) def test_rfc1738_password(self): @@ -53,13 +58,17 @@ class ParseConnectTest(fixtures.TestBase): eq_(u.password, "pass word + other:words") eq_(str(u), "dbtype://user:pass word + other%3Awords@host/dbname") - u = url.make_url('dbtype://username:apples%2Foranges@hostspec/database') + u = url.make_url( + 'dbtype://username:apples%2Foranges@hostspec/database') eq_(u.password, "apples/oranges") eq_(str(u), 'dbtype://username:apples%2Foranges@hostspec/database') - u = url.make_url('dbtype://username:apples%40oranges%40%40@hostspec/database') + u = url.make_url( + 'dbtype://username:apples%40oranges%40%40@hostspec/database') eq_(u.password, "apples@oranges@@") - eq_(str(u), 'dbtype://username:apples%40oranges%40%40@hostspec/database') + eq_( + str(u), + 'dbtype://username:apples%40oranges%40%40@hostspec/database') u = url.make_url('dbtype://username%40:@hostspec/database') eq_(u.password, '') @@ -70,23 +79,23 @@ class ParseConnectTest(fixtures.TestBase): eq_(u.password, 'pass/word') eq_(str(u), 'dbtype://username:pass%2Fword@hostspec/database') + class DialectImportTest(fixtures.TestBase): def test_import_base_dialects(self): - # the globals() somehow makes it for the exec() + nose3. for name in ( - 'mysql', - 'firebird', - 'postgresql', - 'sqlite', - 'oracle', - 'mssql', - ): + 'mysql', + 'firebird', + 'postgresql', + 'sqlite', + 'oracle', + 'mssql'): exec ('from sqlalchemy.dialects import %s\ndialect = ' '%s.dialect()' % (name, name), globals()) eq_(dialect.name, name) + class CreateEngineTest(fixtures.TestBase): """test that create_engine arguments of different types get propagated properly""" @@ -97,26 +106,28 @@ class CreateEngineTest(fixtures.TestBase): create_engine('postgresql://scott:tiger@somehost/test?foobe' 'r=12&lala=18&fooz=somevalue', module=dbapi, _initialize=False) - c = e.connect() + e.connect() def test_kwargs(self): dbapi = MockDBAPI(foober=12, lala=18, hoho={'this': 'dict'}, fooz='somevalue') e = \ - create_engine('postgresql://scott:tiger@somehost/test?fooz=' - 'somevalue', connect_args={'foober': 12, - 'lala': 18, 'hoho': {'this': 'dict'}}, - module=dbapi, _initialize=False) - c = e.connect() - + create_engine( + 'postgresql://scott:tiger@somehost/test?fooz=' + 'somevalue', connect_args={ + 'foober': 12, + 'lala': 18, 'hoho': {'this': 'dict'}}, + module=dbapi, _initialize=False) + e.connect() def test_engine_from_config(self): dbapi = mock_dbapi - config = \ - {'sqlalchemy.url': 'postgresql://scott:tiger@somehost/test'\ - '?fooz=somevalue', 'sqlalchemy.pool_recycle': '50', - 'sqlalchemy.echo': 'true'} + config = { + 'sqlalchemy.url': 'postgresql://scott:tiger@somehost/test' + '?fooz=somevalue', + 'sqlalchemy.pool_recycle': '50', + 'sqlalchemy.echo': 'true'} e = engine_from_config(config, module=dbapi, _initialize=False) assert e.pool._recycle == 50 @@ -125,7 +136,6 @@ class CreateEngineTest(fixtures.TestBase): 'z=somevalue') assert e.echo is True - def test_engine_from_config_custom(self): from sqlalchemy import util from sqlalchemy.dialects import registry @@ -143,8 +153,9 @@ class CreateEngineTest(fixtures.TestBase): global dialect dialect = MyDialect - registry.register("mockdialect.barb", - ".".join(tokens[0:-1]), tokens[-1]) + registry.register( + "mockdialect.barb", + ".".join(tokens[0:-1]), tokens[-1]) config = { "sqlalchemy.url": "mockdialect+barb://", @@ -155,7 +166,6 @@ class CreateEngineTest(fixtures.TestBase): eq_(e.dialect.foobar, 5) eq_(e.dialect.bathoho, False) - def test_custom(self): dbapi = MockDBAPI(foober=12, lala=18, hoho={'this': 'dict'}, fooz='somevalue') @@ -169,7 +179,7 @@ class CreateEngineTest(fixtures.TestBase): e = create_engine('postgresql://', creator=connect, module=dbapi, _initialize=False) - c = e.connect() + e.connect() def test_recycle(self): dbapi = MockDBAPI(foober=12, lala=18, hoho={'this': 'dict'}, @@ -188,8 +198,9 @@ class CreateEngineTest(fixtures.TestBase): (True, pool.reset_rollback), (False, pool.reset_none), ]: - e = create_engine('postgresql://', pool_reset_on_return=value, - module=dbapi, _initialize=False) + e = create_engine( + 'postgresql://', pool_reset_on_return=value, + module=dbapi, _initialize=False) assert e.pool._reset_on_return is expected assert_raises( @@ -217,7 +228,7 @@ class CreateEngineTest(fixtures.TestBase): lala=5, use_ansi=True, module=mock_dbapi, - ) + ) assert_raises(TypeError, create_engine, 'postgresql://', lala=5, module=mock_dbapi) assert_raises(TypeError, create_engine, 'sqlite://', lala=5, @@ -233,14 +244,14 @@ class CreateEngineTest(fixtures.TestBase): dbapi = MockDBAPI() dbapi.Error = sqlite3.Error, dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock(side_effect=sqlite3.ProgrammingError("random error")) + dbapi.connect = Mock( + side_effect=sqlite3.ProgrammingError("random error")) try: create_engine('sqlite://', module=dbapi).connect() assert False except tsa.exc.DBAPIError as de: assert not de.connection_invalidated - @testing.requires.sqlite def test_dont_touch_non_dbapi_exception_on_connect(self): e = create_engine('sqlite://') @@ -260,10 +271,12 @@ class CreateEngineTest(fixtures.TestBase): eq_(is_disconnect.call_count, 0) def test_ensure_dialect_does_is_disconnect_no_conn(self): - """test that is_disconnect() doesn't choke if no connection, cursor given.""" + """test that is_disconnect() doesn't choke if no connection, + cursor given.""" dialect = testing.db.dialect dbapi = dialect.dbapi - assert not dialect.is_disconnect(dbapi.OperationalError("test"), None, None) + assert not dialect.is_disconnect( + dbapi.OperationalError("test"), None, None) @testing.requires.sqlite def test_invalidate_on_connect(self): @@ -280,8 +293,9 @@ class CreateEngineTest(fixtures.TestBase): dbapi = MockDBAPI() dbapi.Error = sqlite3.Error, dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock(side_effect=sqlite3.ProgrammingError( - "Cannot operate on a closed database.")) + dbapi.connect = Mock( + side_effect=sqlite3.ProgrammingError( + "Cannot operate on a closed database.")) try: create_engine('sqlite://', module=dbapi).connect() assert False @@ -313,7 +327,7 @@ class CreateEngineTest(fixtures.TestBase): echo_pool=None, module=mock_dbapi, _initialize=False, - ) + ) assert e.pool._recycle == 50 # these args work for QueuePool @@ -325,7 +339,7 @@ class CreateEngineTest(fixtures.TestBase): poolclass=tsa.pool.QueuePool, module=mock_dbapi, _initialize=False, - ) + ) # but not SingletonThreadPool @@ -338,7 +352,8 @@ class CreateEngineTest(fixtures.TestBase): poolclass=tsa.pool.SingletonThreadPool, module=mock_sqlite_dbapi, _initialize=False, - ) + ) + class TestRegNewDBAPI(fixtures.TestBase): def test_register_base(self): @@ -361,7 +376,8 @@ class TestRegNewDBAPI(fixtures.TestBase): global dialect dialect = MockDialect - registry.register("mockdialect.foob", ".".join(tokens[0:-1]), tokens[-1]) + registry.register( + "mockdialect.foob", ".".join(tokens[0:-1]), tokens[-1]) e = create_engine("mockdialect+foob://") assert isinstance(e.dialect, MockDialect) @@ -373,13 +389,16 @@ class TestRegNewDBAPI(fixtures.TestBase): e = create_engine("mysql+my_mock_dialect://") assert isinstance(e.dialect, MockDialect) + class MockDialect(DefaultDialect): @classmethod def dbapi(cls, **kw): return MockDBAPI() + def MockDBAPI(**assert_kwargs): connection = Mock(get_server_version_info=Mock(return_value='5.0')) + def connect(*args, **kwargs): for k in assert_kwargs: assert k in kwargs, 'key %s not present in dictionary' % k @@ -389,12 +408,12 @@ def MockDBAPI(**assert_kwargs): return connection return MagicMock( - sqlite_version_info=(99, 9, 9,), - version_info=(99, 9, 9,), - sqlite_version='99.9.9', - paramstyle='named', - connect=Mock(side_effect=connect) - ) + sqlite_version_info=(99, 9, 9,), + version_info=(99, 9, 9,), + sqlite_version='99.9.9', + paramstyle='named', + connect=Mock(side_effect=connect) + ) mock_dbapi = MockDBAPI() mock_sqlite_dbapi = msd = MockDBAPI() -- cgit v1.2.1 From 41e7253dee168b8c26c4993d27aac11f98c7f9e3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 12:12:44 -0500 Subject: - The engine-level error handling and wrapping routines will now take effect in all engine connection use cases, including when user-custom connect routines are used via the :paramref:`.create_engine.creator` parameter, as well as when the :class:`.Connection` encounters a connection error on revalidation. fixes #3266 --- doc/build/changelog/changelog_10.rst | 15 +++++ doc/build/changelog/migration_10.rst | 23 +++++++ lib/sqlalchemy/engine/base.py | 74 +++++++++++++++++++++-- lib/sqlalchemy/engine/interfaces.py | 18 +++++- lib/sqlalchemy/engine/strategies.py | 11 +--- lib/sqlalchemy/engine/threadlocal.py | 2 +- lib/sqlalchemy/events.py | 6 ++ test/engine/test_parseconnect.py | 113 ++++++++++++++++++++++++++++++++++- 8 files changed, 243 insertions(+), 19 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index b71ecc15d..b8b513821 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,21 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, engine + :tickets: 3266 + + The engine-level error handling and wrapping routines will now + take effect in all engine connection use cases, including + when user-custom connect routines are used via the + :paramref:`.create_engine.creator` parameter, as well as when + the :class:`.Connection` encounters a connection error on + revalidation. + + .. seealso:: + + :ref:`change_3266` + .. change:: :tags: feature, oracle diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 27a4fae4c..15e066a75 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -872,6 +872,29 @@ labeled uniquely. :ticket:`3170` +.. _change_3266: + +DBAPI exception wrapping and handle_error() event improvements +-------------------------------------------------------------- + +SQLAlchemy's wrapping of DBAPI exceptions was not taking place in the +case where a :class:`.Connection` object was invalidated, and then tried +to reconnect and encountered an error; this has been resolved. + +Additionally, the recently added :meth:`.ConnectionEvents.handle_error` +event is now invoked for errors that occur upon initial connect, upon +reconnect, and when :func:`.create_engine` is used given a custom connection +function via :paramref:`.create_engine.creator`. + +The :class:`.ExceptionContext` object has a new datamember +:attr:`.ExceptionContext.engine` that will always refer to the :class:`.Engine` +in use, in those cases when the :class:`.Connection` object is not available +(e.g. on initial connect). + + +:ticket:`3266` + + .. _behavioral_changes_orm_10: Behavioral Changes - ORM diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index dd82be1d1..901ab07eb 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -276,7 +276,7 @@ class Connection(Connectable): raise exc.InvalidRequestError( "Can't reconnect until invalid " "transaction is rolled back") - self.__connection = self.engine.raw_connection() + self.__connection = self.engine.raw_connection(self) self.__invalid = False return self.__connection raise exc.ResourceClosedError("This Connection is closed") @@ -1194,7 +1194,8 @@ class Connection(Connectable): # new handle_error event ctx = ExceptionContextImpl( - e, sqlalchemy_exception, self, cursor, statement, + e, sqlalchemy_exception, self.engine, + self, cursor, statement, parameters, context, self._is_disconnect) for fn in self.dispatch.handle_error: @@ -1242,6 +1243,58 @@ class Connection(Connectable): if self.should_close_with_result: self.close() + @classmethod + def _handle_dbapi_exception_noconnection( + cls, e, dialect, engine, connection): + exc_info = sys.exc_info() + + is_disconnect = dialect.is_disconnect(e, None, None) + + should_wrap = isinstance(e, dialect.dbapi.Error) + + if should_wrap: + sqlalchemy_exception = exc.DBAPIError.instance( + None, + None, + e, + dialect.dbapi.Error, + connection_invalidated=is_disconnect) + else: + sqlalchemy_exception = None + + newraise = None + + if engine._has_events: + ctx = ExceptionContextImpl( + e, sqlalchemy_exception, engine, connection, None, None, + None, None, is_disconnect) + for fn in engine.dispatch.handle_error: + try: + # handler returns an exception; + # call next handler in a chain + per_fn = fn(ctx) + if per_fn is not None: + ctx.chained_exception = newraise = per_fn + except Exception as _raised: + # handler raises an exception - stop processing + newraise = _raised + break + + if sqlalchemy_exception and \ + is_disconnect != ctx.is_disconnect: + sqlalchemy_exception.connection_invalidated = \ + is_disconnect = ctx.is_disconnect + + if newraise: + util.raise_from_cause(newraise, exc_info) + elif should_wrap: + util.raise_from_cause( + sqlalchemy_exception, + exc_info + ) + else: + util.reraise(*exc_info) + def default_schema_name(self): return self.engine.dialect.get_default_schema_name(self) @@ -1320,8 +1373,9 @@ class ExceptionContextImpl(ExceptionContext): """Implement the :class:`.ExceptionContext` interface.""" def __init__(self, exception, sqlalchemy_exception, - connection, cursor, statement, parameters, + engine, connection, cursor, statement, parameters, context, is_disconnect): + self.engine = engine self.connection = connection self.sqlalchemy_exception = sqlalchemy_exception self.original_exception = exception @@ -1898,7 +1952,15 @@ class Engine(Connectable, log.Identified): """ return self.run_callable(self.dialect.has_table, table_name, schema) - def raw_connection(self): + def _wrap_pool_connect(self, fn, connection=None): + dialect = self.dialect + try: + return fn() + except dialect.dbapi.Error as e: + Connection._handle_dbapi_exception_noconnection( + e, dialect, self, connection) + + def raw_connection(self, _connection=None): """Return a "raw" DBAPI connection from the connection pool. The returned object is a proxied version of the DBAPI @@ -1914,8 +1976,8 @@ class Engine(Connectable, log.Identified): :meth:`.Engine.connect` method. """ - - return self.pool.unique_connection() + return self._wrap_pool_connect( + self.pool.unique_connection, _connection) class OptionEngine(Engine): diff --git a/lib/sqlalchemy/engine/interfaces.py b/lib/sqlalchemy/engine/interfaces.py index 0ad2efae0..5f66e54b5 100644 --- a/lib/sqlalchemy/engine/interfaces.py +++ b/lib/sqlalchemy/engine/interfaces.py @@ -917,7 +917,23 @@ class ExceptionContext(object): connection = None """The :class:`.Connection` in use during the exception. - This member is always present. + This member is present, except in the case of a failure when + first connecting. + + .. seealso:: + + :attr:`.ExceptionContext.engine` + + + """ + + engine = None + """The :class:`.Engine` in use during the exception. + + This member should always be present, even in the case of a failure + when first connecting. + + .. versionadded:: 1.0.0 """ diff --git a/lib/sqlalchemy/engine/strategies.py b/lib/sqlalchemy/engine/strategies.py index 398ef8df6..fd665ad03 100644 --- a/lib/sqlalchemy/engine/strategies.py +++ b/lib/sqlalchemy/engine/strategies.py @@ -86,16 +86,7 @@ class DefaultEngineStrategy(EngineStrategy): pool = pop_kwarg('pool', None) if pool is None: def connect(): - try: - return dialect.connect(*cargs, **cparams) - except dialect.dbapi.Error as e: - invalidated = dialect.is_disconnect(e, None, None) - util.raise_from_cause( - exc.DBAPIError.instance( - None, None, e, dialect.dbapi.Error, - connection_invalidated=invalidated - ) - ) + return dialect.connect(*cargs, **cparams) creator = pop_kwarg('creator', connect) diff --git a/lib/sqlalchemy/engine/threadlocal.py b/lib/sqlalchemy/engine/threadlocal.py index 637523a0e..71caac626 100644 --- a/lib/sqlalchemy/engine/threadlocal.py +++ b/lib/sqlalchemy/engine/threadlocal.py @@ -59,7 +59,7 @@ class TLEngine(base.Engine): # guards against pool-level reapers, if desired. # or not connection.connection.is_valid: connection = self._tl_connection_cls( - self, self.pool.connect(), **kw) + self, self._wrap_pool_connect(self.pool.connect), **kw) self._connections.conn = weakref.ref(connection) return connection._increment_connect() diff --git a/lib/sqlalchemy/events.py b/lib/sqlalchemy/events.py index c144902cd..8600c20f5 100644 --- a/lib/sqlalchemy/events.py +++ b/lib/sqlalchemy/events.py @@ -739,6 +739,12 @@ class ConnectionEvents(event.Events): .. versionadded:: 0.9.7 Added the :meth:`.ConnectionEvents.handle_error` hook. + .. versionchanged:: 1.0.0 The :meth:`.handle_error` event is now + invoked when an :class:`.Engine` fails during the initial + call to :meth:`.Engine.connect`, as well as when a + :class:`.Connection` object encounters an error during a + reconnect operation. + .. versionchanged:: 1.0.0 The :meth:`.handle_error` event is not fired off when a dialect makes use of the ``skip_user_error_events`` execution option. This is used diff --git a/test/engine/test_parseconnect.py b/test/engine/test_parseconnect.py index d8f202f99..72a089aca 100644 --- a/test/engine/test_parseconnect.py +++ b/test/engine/test_parseconnect.py @@ -6,6 +6,7 @@ import sqlalchemy as tsa from sqlalchemy.testing import fixtures from sqlalchemy import testing from sqlalchemy.testing.mock import Mock, MagicMock +from sqlalchemy import event dialect = None @@ -240,7 +241,6 @@ class CreateEngineTest(fixtures.TestBase): def test_wraps_connect_in_dbapi(self): e = create_engine('sqlite://') sqlite3 = e.dialect.dbapi - dbapi = MockDBAPI() dbapi.Error = sqlite3.Error, dbapi.ProgrammingError = sqlite3.ProgrammingError @@ -252,6 +252,117 @@ class CreateEngineTest(fixtures.TestBase): except tsa.exc.DBAPIError as de: assert not de.connection_invalidated + @testing.requires.sqlite + def test_handle_error_event_connect(self): + e = create_engine('sqlite://') + dbapi = MockDBAPI() + sqlite3 = e.dialect.dbapi + dbapi.Error = sqlite3.Error, + dbapi.ProgrammingError = sqlite3.ProgrammingError + dbapi.connect = Mock( + side_effect=sqlite3.ProgrammingError("random error")) + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is None + raise MySpecialException("failed operation") + + assert_raises( + MySpecialException, + eng.connect + ) + + @testing.requires.sqlite + def test_handle_error_event_reconnect(self): + e = create_engine('sqlite://') + dbapi = MockDBAPI() + sqlite3 = e.dialect.dbapi + dbapi.Error = sqlite3.Error, + dbapi.ProgrammingError = sqlite3.ProgrammingError + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi, _initialize=False) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is conn + raise MySpecialException("failed operation") + + conn = eng.connect() + conn.invalidate() + + dbapi.connect = Mock( + side_effect=sqlite3.ProgrammingError("random error")) + + assert_raises( + MySpecialException, + conn._revalidate_connection + ) + + @testing.requires.sqlite + def test_handle_error_custom_connect(self): + e = create_engine('sqlite://') + + dbapi = MockDBAPI() + sqlite3 = e.dialect.dbapi + dbapi.Error = sqlite3.Error, + dbapi.ProgrammingError = sqlite3.ProgrammingError + + class MySpecialException(Exception): + pass + + def custom_connect(): + raise sqlite3.ProgrammingError("random error") + + eng = create_engine('sqlite://', module=dbapi, creator=custom_connect) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is None + raise MySpecialException("failed operation") + + assert_raises( + MySpecialException, + eng.connect + ) + + @testing.requires.sqlite + def test_handle_error_event_connect_invalidate_flag(self): + e = create_engine('sqlite://') + dbapi = MockDBAPI() + sqlite3 = e.dialect.dbapi + dbapi.Error = sqlite3.Error, + dbapi.ProgrammingError = sqlite3.ProgrammingError + dbapi.connect = Mock( + side_effect=sqlite3.ProgrammingError( + "Cannot operate on a closed database.")) + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.is_disconnect + ctx.is_disconnect = False + + try: + eng.connect() + assert False + except tsa.exc.DBAPIError as de: + assert not de.connection_invalidated + @testing.requires.sqlite def test_dont_touch_non_dbapi_exception_on_connect(self): e = create_engine('sqlite://') -- cgit v1.2.1 From d204e61f63756f2bbd3322377a283fc995e562ec Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 12:18:11 -0500 Subject: - document / work around that dialect_options isn't necessarily there --- lib/sqlalchemy/engine/reflection.py | 5 ++++- lib/sqlalchemy/testing/suite/test_reflection.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/engine/reflection.py b/lib/sqlalchemy/engine/reflection.py index ebc96f5dd..25f084c15 100644 --- a/lib/sqlalchemy/engine/reflection.py +++ b/lib/sqlalchemy/engine/reflection.py @@ -395,7 +395,10 @@ class Inspector(object): boolean dialect_options - dict of dialect-specific index options + dict of dialect-specific index options. May not be present + for all dialects. + + .. versionadded:: 1.0.0 :param table_name: string name of the table. For special quoting, use :class:`.quoted_name`. diff --git a/lib/sqlalchemy/testing/suite/test_reflection.py b/lib/sqlalchemy/testing/suite/test_reflection.py index e58b6f068..3edbdeb8c 100644 --- a/lib/sqlalchemy/testing/suite/test_reflection.py +++ b/lib/sqlalchemy/testing/suite/test_reflection.py @@ -515,6 +515,8 @@ class ComponentReflectionTest(fixtures.TablesTest): def test_get_temp_table_indexes(self): insp = inspect(self.metadata.bind) indexes = insp.get_indexes('user_tmp') + for ind in indexes: + ind.pop('dialect_options', None) eq_( # TODO: we need to add better filtering for indexes/uq constraints # that are doubled up -- cgit v1.2.1 From ec6214457ed71f0ae87d83076e084214650aae5d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 14:19:36 -0500 Subject: - pep8 --- test/dialect/test_sqlite.py | 423 ++++++++++++++++++++++++-------------------- 1 file changed, 232 insertions(+), 191 deletions(-) diff --git a/test/dialect/test_sqlite.py b/test/dialect/test_sqlite.py index 124208dbe..04e82e686 100644 --- a/test/dialect/test_sqlite.py +++ b/test/dialect/test_sqlite.py @@ -7,8 +7,8 @@ import datetime from sqlalchemy.testing import eq_, assert_raises, \ assert_raises_message, is_ from sqlalchemy import Table, select, bindparam, Column,\ - MetaData, func, extract, ForeignKey, text, DefaultClause, and_, create_engine,\ - UniqueConstraint + MetaData, func, extract, ForeignKey, text, DefaultClause, and_, \ + create_engine, UniqueConstraint from sqlalchemy.types import Integer, String, Boolean, DateTime, Date, Time from sqlalchemy import types as sqltypes from sqlalchemy import event, inspect @@ -21,6 +21,8 @@ from sqlalchemy.testing import fixtures, AssertsCompiledSQL, \ AssertsExecutionResults, engines from sqlalchemy import testing from sqlalchemy.schema import CreateTable +from sqlalchemy.engine.reflection import Inspector + class TestTypes(fixtures.TestBase, AssertsExecutionResults): @@ -32,9 +34,10 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): """ meta = MetaData(testing.db) - t = Table('bool_table', meta, Column('id', Integer, - primary_key=True), Column('boo', - Boolean(create_constraint=False))) + t = Table( + 'bool_table', meta, + Column('id', Integer, primary_key=True), + Column('boo', Boolean(create_constraint=False))) try: meta.create_all() testing.db.execute("INSERT INTO bool_table (id, boo) " @@ -69,28 +72,31 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): ValueError, "Couldn't parse %s string." % disp, lambda: testing.db.execute( - text("select 'ASDF' as value", typemap={"value":typ}) + text("select 'ASDF' as value", typemap={"value": typ}) ).scalar() ) def test_native_datetime(self): dbapi = testing.db.dialect.dbapi - connect_args = {'detect_types': dbapi.PARSE_DECLTYPES \ - | dbapi.PARSE_COLNAMES} - engine = engines.testing_engine(options={'connect_args' - : connect_args, 'native_datetime': True}) - t = Table('datetest', MetaData(), - Column('id', Integer, primary_key=True), - Column('d1', Date), Column('d2', sqltypes.TIMESTAMP)) + connect_args = { + 'detect_types': dbapi.PARSE_DECLTYPES | dbapi.PARSE_COLNAMES} + engine = engines.testing_engine( + options={'connect_args': connect_args, 'native_datetime': True}) + t = Table( + 'datetest', MetaData(), + Column('id', Integer, primary_key=True), + Column('d1', Date), Column('d2', sqltypes.TIMESTAMP)) t.create(engine) try: - engine.execute(t.insert(), {'d1': datetime.date(2010, 5, - 10), - 'd2': datetime.datetime( 2010, 5, 10, 12, 15, 25, - )}) + engine.execute(t.insert(), { + 'd1': datetime.date(2010, 5, 10), + 'd2': datetime.datetime(2010, 5, 10, 12, 15, 25) + }) row = engine.execute(t.select()).first() - eq_(row, (1, datetime.date(2010, 5, 10), - datetime.datetime( 2010, 5, 10, 12, 15, 25, ))) + eq_( + row, + (1, datetime.date(2010, 5, 10), + datetime.datetime(2010, 5, 10, 12, 15, 25))) r = engine.execute(func.current_date()).scalar() assert isinstance(r, util.string_types) finally: @@ -100,15 +106,16 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): @testing.provide_metadata def test_custom_datetime(self): sqlite_date = sqlite.DATETIME( - # 2004-05-21T00:00:00 - storage_format="%(year)04d-%(month)02d-%(day)02d" - "T%(hour)02d:%(minute)02d:%(second)02d", - regexp=r"(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)", - ) + # 2004-05-21T00:00:00 + storage_format="%(year)04d-%(month)02d-%(day)02d" + "T%(hour)02d:%(minute)02d:%(second)02d", + regexp=r"(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)", + ) t = Table('t', self.metadata, Column('d', sqlite_date)) self.metadata.create_all(testing.db) - testing.db.execute(t.insert(). - values(d=datetime.datetime(2010, 10, 15, 12, 37, 0))) + testing.db.execute( + t.insert(). + values(d=datetime.datetime(2010, 10, 15, 12, 37, 0))) testing.db.execute("insert into t (d) values ('2004-05-21T00:00:00')") eq_( testing.db.execute("select * from t order by d").fetchall(), @@ -116,21 +123,23 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): ) eq_( testing.db.execute(select([t.c.d]).order_by(t.c.d)).fetchall(), - [(datetime.datetime(2004, 5, 21, 0, 0),), - (datetime.datetime(2010, 10, 15, 12, 37),)] + [ + (datetime.datetime(2004, 5, 21, 0, 0),), + (datetime.datetime(2010, 10, 15, 12, 37),)] ) @testing.provide_metadata def test_custom_date(self): sqlite_date = sqlite.DATE( - # 2004-05-21T00:00:00 - storage_format="%(year)04d|%(month)02d|%(day)02d", - regexp=r"(\d+)\|(\d+)\|(\d+)", - ) + # 2004-05-21T00:00:00 + storage_format="%(year)04d|%(month)02d|%(day)02d", + regexp=r"(\d+)\|(\d+)\|(\d+)", + ) t = Table('t', self.metadata, Column('d', sqlite_date)) self.metadata.create_all(testing.db) - testing.db.execute(t.insert(). - values(d=datetime.date(2010, 10, 15))) + testing.db.execute( + t.insert(). + values(d=datetime.date(2010, 10, 15))) testing.db.execute("insert into t (d) values ('2004|05|21')") eq_( testing.db.execute("select * from t order by d").fetchall(), @@ -138,11 +147,11 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): ) eq_( testing.db.execute(select([t.c.d]).order_by(t.c.d)).fetchall(), - [(datetime.date(2004, 5, 21),), - (datetime.date(2010, 10, 15),)] + [ + (datetime.date(2004, 5, 21),), + (datetime.date(2010, 10, 15),)] ) - def test_no_convert_unicode(self): """test no utf-8 encoding occurs""" @@ -156,7 +165,7 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): sqltypes.CHAR(convert_unicode=True), sqltypes.Unicode(), sqltypes.UnicodeText(), - ): + ): bindproc = t.dialect_impl(dialect).bind_processor(dialect) assert not bindproc or \ isinstance(bindproc(util.u('some string')), util.text_type) @@ -198,6 +207,7 @@ class DateTimeTest(fixtures.TestBase, AssertsCompiledSQL): rp = sldt.result_processor(None, None) eq_(rp(bp(dt)), dt) + class DateTest(fixtures.TestBase, AssertsCompiledSQL): def test_default(self): @@ -221,6 +231,7 @@ class DateTest(fixtures.TestBase, AssertsCompiledSQL): rp = sldt.result_processor(None, None) eq_(rp(bp(dt)), dt) + class TimeTest(fixtures.TestBase, AssertsCompiledSQL): def test_default(self): @@ -333,8 +344,9 @@ class DefaultsTest(fixtures.TestBase, AssertsCompiledSQL): @testing.provide_metadata def test_boolean_default(self): - t = Table("t", self.metadata, - Column("x", Boolean, server_default=sql.false())) + t = Table( + "t", self.metadata, + Column("x", Boolean, server_default=sql.false())) t.create(testing.db) testing.db.execute(t.insert()) testing.db.execute(t.insert().values(x=True)) @@ -351,7 +363,6 @@ class DefaultsTest(fixtures.TestBase, AssertsCompiledSQL): eq_(info['default'], '3') - class DialectTest(fixtures.TestBase, AssertsExecutionResults): __only_on__ = 'sqlite' @@ -372,7 +383,7 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): Column('true', Integer), Column('false', Integer), Column('column', Integer), - ) + ) try: meta.create_all() t.insert().execute(safe=1) @@ -403,8 +414,8 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): table1 = Table('django_admin_log', metadata, autoload=True) table2 = Table('django_content_type', metadata, autoload=True) j = table1.join(table2) - assert j.onclause.compare(table1.c.content_type_id - == table2.c.id) + assert j.onclause.compare( + table1.c.content_type_id == table2.c.id) @testing.provide_metadata def test_quoted_identifiers_functional_two(self): @@ -426,8 +437,8 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): # unfortunately, still can't do this; sqlite quadruples # up the quotes on the table name here for pragma foreign_key_list - #testing.db.execute(r''' - #CREATE TABLE """b""" ( + # testing.db.execute(r''' + # CREATE TABLE """b""" ( # """id""" integer NOT NULL PRIMARY KEY, # """aid""" integer NULL # REFERENCES """a""" ("""id""") @@ -439,14 +450,13 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): #table2 = Table(r'"b"', metadata, autoload=True) #j = table1.join(table2) - #assert j.onclause.compare(table1.c['"id"'] + # assert j.onclause.compare(table1.c['"id"'] # == table2.c['"aid"']) def test_legacy_quoted_identifiers_unit(self): dialect = sqlite.dialect() dialect._broken_fk_pragma_quotes = True - for row in [ (0, 'target', 'tid', 'id'), (0, '"target"', 'tid', 'id'), @@ -457,7 +467,9 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): fks = {} fkeys = [] dialect._parse_fk(fks, fkeys, *row) - eq_(fkeys, [{ + eq_( + fkeys, + [{ 'referred_table': 'target', 'referred_columns': ['id'], 'referred_schema': None, @@ -470,17 +482,17 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): # amazingly, pysqlite seems to still deliver cursor.description # as encoded bytes in py2k - t = Table('x', self.metadata, - Column(u('méil'), Integer, primary_key=True), - Column(ue('\u6e2c\u8a66'), Integer), - ) + t = Table( + 'x', self.metadata, + Column(u('méil'), Integer, primary_key=True), + Column(ue('\u6e2c\u8a66'), Integer), + ) self.metadata.create_all(testing.db) result = testing.db.execute(t.select()) assert u('méil') in result.keys() assert ue('\u6e2c\u8a66') in result.keys() - def test_file_path_is_absolute(self): d = pysqlite_dialect.dialect() eq_( @@ -498,48 +510,51 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): e = create_engine('sqlite+pysqlite:///foo.db') assert e.pool.__class__ is pool.NullPool + @testing.provide_metadata def test_dont_reflect_autoindex(self): - meta = MetaData(testing.db) - t = Table('foo', meta, Column('bar', String, primary_key=True)) + meta = self.metadata + Table('foo', meta, Column('bar', String, primary_key=True)) meta.create_all() - from sqlalchemy.engine.reflection import Inspector - try: - inspector = Inspector(testing.db) - eq_(inspector.get_indexes('foo'), []) - eq_(inspector.get_indexes('foo', - include_auto_indexes=True), [{'unique': 1, 'name' - : 'sqlite_autoindex_foo_1', 'column_names': ['bar']}]) - finally: - meta.drop_all() + inspector = Inspector(testing.db) + eq_(inspector.get_indexes('foo'), []) + eq_( + inspector.get_indexes('foo', include_auto_indexes=True), + [{ + 'unique': 1, + 'name': 'sqlite_autoindex_foo_1', + 'column_names': ['bar']}]) + @testing.provide_metadata def test_create_index_with_schema(self): """Test creation of index with explicit schema""" - meta = MetaData(testing.db) - t = Table('foo', meta, Column('bar', String, index=True), - schema='main') - try: - meta.create_all() - finally: - meta.drop_all() + meta = self.metadata + Table( + 'foo', meta, Column('bar', String, index=True), + schema='main') + meta.create_all() + inspector = Inspector(testing.db) + eq_( + inspector.get_indexes('foo', schema='main'), + [{'unique': 0, 'name': u'ix_main_foo_bar', + 'column_names': [u'bar']}]) + @testing.provide_metadata def test_get_unique_constraints(self): - meta = MetaData(testing.db) - t1 = Table('foo', meta, Column('f', Integer), - UniqueConstraint('f', name='foo_f')) - t2 = Table('bar', meta, Column('b', Integer), - UniqueConstraint('b', name='bar_b'), - prefixes=['TEMPORARY']) + meta = self.metadata + Table( + 'foo', meta, Column('f', Integer), + UniqueConstraint('f', name='foo_f')) + Table( + 'bar', meta, Column('b', Integer), + UniqueConstraint('b', name='bar_b'), + prefixes=['TEMPORARY']) meta.create_all() - from sqlalchemy.engine.reflection import Inspector - try: - inspector = Inspector(testing.db) - eq_(inspector.get_unique_constraints('foo'), - [{'column_names': [u'f'], 'name': u'foo_f'}]) - eq_(inspector.get_unique_constraints('bar'), - [{'column_names': [u'b'], 'name': u'bar_b'}]) - finally: - meta.drop_all() + inspector = Inspector(testing.db) + eq_(inspector.get_unique_constraints('foo'), + [{'column_names': [u'f'], 'name': u'foo_f'}]) + eq_(inspector.get_unique_constraints('bar'), + [{'column_names': [u'b'], 'name': u'bar_b'}]) class AttachedMemoryDBTest(fixtures.TestBase): @@ -662,7 +677,7 @@ class SQLTest(fixtures.TestBase, AssertsCompiledSQL): 'epoch': '%s', 'dow': '%w', 'week': '%W', - } + } for field, subst in mapping.items(): self.assert_compile(select([extract(field, t.c.col1)]), "SELECT CAST(STRFTIME('%s', t.col1) AS " @@ -685,53 +700,57 @@ class SQLTest(fixtures.TestBase, AssertsCompiledSQL): def test_constraints_with_schemas(self): metadata = MetaData() - t1 = Table('t1', metadata, - Column('id', Integer, primary_key=True), - schema='master') - t2 = Table('t2', metadata, - Column('id', Integer, primary_key=True), - Column('t1_id', Integer, ForeignKey('master.t1.id')), - schema='master' - ) - t3 = Table('t3', metadata, - Column('id', Integer, primary_key=True), - Column('t1_id', Integer, ForeignKey('master.t1.id')), - schema='alternate' - ) - t4 = Table('t4', metadata, - Column('id', Integer, primary_key=True), - Column('t1_id', Integer, ForeignKey('master.t1.id')), - ) + Table( + 't1', metadata, + Column('id', Integer, primary_key=True), + schema='master') + t2 = Table( + 't2', metadata, + Column('id', Integer, primary_key=True), + Column('t1_id', Integer, ForeignKey('master.t1.id')), + schema='master' + ) + t3 = Table( + 't3', metadata, + Column('id', Integer, primary_key=True), + Column('t1_id', Integer, ForeignKey('master.t1.id')), + schema='alternate' + ) + t4 = Table( + 't4', metadata, + Column('id', Integer, primary_key=True), + Column('t1_id', Integer, ForeignKey('master.t1.id')), + ) # schema->schema, generate REFERENCES with no schema name self.assert_compile( schema.CreateTable(t2), - "CREATE TABLE master.t2 (" - "id INTEGER NOT NULL, " - "t1_id INTEGER, " - "PRIMARY KEY (id), " - "FOREIGN KEY(t1_id) REFERENCES t1 (id)" - ")" + "CREATE TABLE master.t2 (" + "id INTEGER NOT NULL, " + "t1_id INTEGER, " + "PRIMARY KEY (id), " + "FOREIGN KEY(t1_id) REFERENCES t1 (id)" + ")" ) # schema->different schema, don't generate REFERENCES self.assert_compile( schema.CreateTable(t3), - "CREATE TABLE alternate.t3 (" - "id INTEGER NOT NULL, " - "t1_id INTEGER, " - "PRIMARY KEY (id)" - ")" + "CREATE TABLE alternate.t3 (" + "id INTEGER NOT NULL, " + "t1_id INTEGER, " + "PRIMARY KEY (id)" + ")" ) # same for local schema self.assert_compile( schema.CreateTable(t4), - "CREATE TABLE t4 (" - "id INTEGER NOT NULL, " - "t1_id INTEGER, " - "PRIMARY KEY (id)" - ")" + "CREATE TABLE t4 (" + "id INTEGER NOT NULL, " + "t1_id INTEGER, " + "PRIMARY KEY (id)" + ")" ) @@ -756,30 +775,37 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): @testing.exclude('sqlite', '<', (3, 3, 8), 'no database support') def test_empty_insert_pk1(self): - self._test_empty_insert(Table('a', MetaData(testing.db), - Column('id', Integer, - primary_key=True))) + self._test_empty_insert( + Table( + 'a', MetaData(testing.db), + Column('id', Integer, primary_key=True))) @testing.exclude('sqlite', '<', (3, 3, 8), 'no database support') def test_empty_insert_pk2(self): - assert_raises(exc.DBAPIError, self._test_empty_insert, Table('b' - , MetaData(testing.db), Column('x', Integer, - primary_key=True), Column('y', Integer, - primary_key=True))) + assert_raises( + exc.DBAPIError, self._test_empty_insert, + Table( + 'b', MetaData(testing.db), + Column('x', Integer, primary_key=True), + Column('y', Integer, primary_key=True))) @testing.exclude('sqlite', '<', (3, 3, 8), 'no database support') def test_empty_insert_pk3(self): - assert_raises(exc.DBAPIError, self._test_empty_insert, Table('c' - , MetaData(testing.db), Column('x', Integer, - primary_key=True), Column('y', Integer, - DefaultClause('123'), primary_key=True))) + assert_raises( + exc.DBAPIError, self._test_empty_insert, + Table( + 'c', MetaData(testing.db), + Column('x', Integer, primary_key=True), + Column('y', Integer, DefaultClause('123'), primary_key=True))) @testing.exclude('sqlite', '<', (3, 3, 8), 'no database support') def test_empty_insert_pk4(self): - self._test_empty_insert(Table('d', MetaData(testing.db), - Column('x', Integer, primary_key=True), - Column('y', Integer, DefaultClause('123' - )))) + self._test_empty_insert( + Table( + 'd', MetaData(testing.db), + Column('x', Integer, primary_key=True), + Column('y', Integer, DefaultClause('123')) + )) @testing.exclude('sqlite', '<', (3, 3, 8), 'no database support') def test_empty_insert_nopk1(self): @@ -788,9 +814,10 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): @testing.exclude('sqlite', '<', (3, 3, 8), 'no database support') def test_empty_insert_nopk2(self): - self._test_empty_insert(Table('f', MetaData(testing.db), - Column('x', Integer), Column('y', - Integer))) + self._test_empty_insert( + Table( + 'f', MetaData(testing.db), + Column('x', Integer), Column('y', Integer))) def test_inserts_with_spaces(self): tbl = Table('tbl', MetaData('sqlite:///'), Column('with space', @@ -800,8 +827,8 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): tbl.insert().execute({'without': 123}) assert list(tbl.select().execute()) == [(None, 123)] tbl.insert().execute({'with space': 456}) - assert list(tbl.select().execute()) == [(None, 123), (456, - None)] + assert list(tbl.select().execute()) == [ + (None, 123), (456, None)] finally: tbl.drop() @@ -817,6 +844,8 @@ def full_text_search_missing(): except: return True +metadata = cattable = matchtable = None + class MatchTest(fixtures.TestBase, AssertsCompiledSQL): @@ -845,19 +874,20 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): """) matchtable = Table('matchtable', metadata, autoload=True) metadata.create_all() - cattable.insert().execute([{'id': 1, 'description': 'Python'}, - {'id': 2, 'description': 'Ruby'}]) - matchtable.insert().execute([{'id': 1, 'title' - : 'Agile Web Development with Rails' - , 'category_id': 2}, {'id': 2, - 'title': 'Dive Into Python', - 'category_id': 1}, {'id': 3, 'title' - : "Programming Matz's Ruby", - 'category_id': 2}, {'id': 4, 'title' - : 'The Definitive Guide to Django', - 'category_id': 1}, {'id': 5, 'title' - : 'Python in a Nutshell', - 'category_id': 1}]) + cattable.insert().execute( + [{'id': 1, 'description': 'Python'}, + {'id': 2, 'description': 'Ruby'}]) + matchtable.insert().execute( + [ + {'id': 1, 'title': 'Agile Web Development with Rails', + 'category_id': 2}, + {'id': 2, 'title': 'Dive Into Python', 'category_id': 1}, + {'id': 3, 'title': "Programming Matz's Ruby", + 'category_id': 2}, + {'id': 4, 'title': 'The Definitive Guide to Django', + 'category_id': 1}, + {'id': 5, 'title': 'Python in a Nutshell', 'category_id': 1} + ]) @classmethod def teardown_class(cls): @@ -869,35 +899,38 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): def test_simple_match(self): results = \ - matchtable.select().where(matchtable.c.title.match('python' - )).order_by(matchtable.c.id).execute().fetchall() + matchtable.select().where( + matchtable.c.title.match('python')).\ + order_by(matchtable.c.id).execute().fetchall() eq_([2, 5], [r.id for r in results]) def test_simple_prefix_match(self): results = \ - matchtable.select().where(matchtable.c.title.match('nut*' - )).execute().fetchall() + matchtable.select().where( + matchtable.c.title.match('nut*')).execute().fetchall() eq_([5], [r.id for r in results]) def test_or_match(self): results2 = \ matchtable.select().where( - matchtable.c.title.match('nutshell OR ruby' - )).order_by(matchtable.c.id).execute().fetchall() + matchtable.c.title.match('nutshell OR ruby')).\ + order_by(matchtable.c.id).execute().fetchall() eq_([3, 5], [r.id for r in results2]) def test_and_match(self): results2 = \ matchtable.select().where( - matchtable.c.title.match('python nutshell' - )).execute().fetchall() + matchtable.c.title.match('python nutshell') + ).execute().fetchall() eq_([5], [r.id for r in results2]) def test_match_across_joins(self): - results = matchtable.select().where(and_(cattable.c.id - == matchtable.c.category_id, - cattable.c.description.match('Ruby' - ))).order_by(matchtable.c.id).execute().fetchall() + results = matchtable.select().where( + and_( + cattable.c.id == matchtable.c.category_id, + cattable.c.description.match('Ruby') + ) + ).order_by(matchtable.c.id).execute().fetchall() eq_([1, 3], [r.id for r in results]) @@ -907,10 +940,11 @@ class AutoIncrementTest(fixtures.TestBase, AssertsCompiledSQL): table = Table('autoinctable', MetaData(), Column('id', Integer, primary_key=True), Column('x', Integer, default=None), sqlite_autoincrement=True) - self.assert_compile(schema.CreateTable(table), - 'CREATE TABLE autoinctable (id INTEGER NOT ' - 'NULL PRIMARY KEY AUTOINCREMENT, x INTEGER)' - , dialect=sqlite.dialect()) + self.assert_compile( + schema.CreateTable(table), + 'CREATE TABLE autoinctable (id INTEGER NOT ' + 'NULL PRIMARY KEY AUTOINCREMENT, x INTEGER)', + dialect=sqlite.dialect()) def test_sqlite_autoincrement_constraint(self): table = Table( @@ -920,7 +954,7 @@ class AutoIncrementTest(fixtures.TestBase, AssertsCompiledSQL): Column('x', Integer, default=None), UniqueConstraint('x'), sqlite_autoincrement=True, - ) + ) self.assert_compile(schema.CreateTable(table), 'CREATE TABLE autoinctable (id INTEGER NOT ' 'NULL PRIMARY KEY AUTOINCREMENT, x ' @@ -944,7 +978,7 @@ class AutoIncrementTest(fixtures.TestBase, AssertsCompiledSQL): MetaData(), Column('id', MyInteger, primary_key=True), sqlite_autoincrement=True, - ) + ) self.assert_compile(schema.CreateTable(table), 'CREATE TABLE autoinctable (id INTEGER NOT ' 'NULL PRIMARY KEY AUTOINCREMENT)', @@ -958,7 +992,8 @@ class ReflectHeadlessFKsTest(fixtures.TestBase): testing.db.execute("CREATE TABLE a (id INTEGER PRIMARY KEY)") # this syntax actually works on other DBs perhaps we'd want to add # tests to test_reflection - testing.db.execute("CREATE TABLE b (id INTEGER PRIMARY KEY REFERENCES a)") + testing.db.execute( + "CREATE TABLE b (id INTEGER PRIMARY KEY REFERENCES a)") def teardown(self): testing.db.execute("drop table b") @@ -971,21 +1006,24 @@ class ReflectHeadlessFKsTest(fixtures.TestBase): assert b.c.id.references(a.c.id) + class ReflectFKConstraintTest(fixtures.TestBase): __only_on__ = 'sqlite' def setup(self): testing.db.execute("CREATE TABLE a1 (id INTEGER PRIMARY KEY)") testing.db.execute("CREATE TABLE a2 (id INTEGER PRIMARY KEY)") - testing.db.execute("CREATE TABLE b (id INTEGER PRIMARY KEY, " - "FOREIGN KEY(id) REFERENCES a1(id)," - "FOREIGN KEY(id) REFERENCES a2(id)" - ")") - testing.db.execute("CREATE TABLE c (id INTEGER, " - "CONSTRAINT bar PRIMARY KEY(id)," - "CONSTRAINT foo1 FOREIGN KEY(id) REFERENCES a1(id)," - "CONSTRAINT foo2 FOREIGN KEY(id) REFERENCES a2(id)" - ")") + testing.db.execute( + "CREATE TABLE b (id INTEGER PRIMARY KEY, " + "FOREIGN KEY(id) REFERENCES a1(id)," + "FOREIGN KEY(id) REFERENCES a2(id)" + ")") + testing.db.execute( + "CREATE TABLE c (id INTEGER, " + "CONSTRAINT bar PRIMARY KEY(id)," + "CONSTRAINT foo1 FOREIGN KEY(id) REFERENCES a1(id)," + "CONSTRAINT foo2 FOREIGN KEY(id) REFERENCES a2(id)" + ")") def teardown(self): testing.db.execute("drop table c") @@ -1005,7 +1043,8 @@ class ReflectFKConstraintTest(fixtures.TestBase): def test_name_not_none(self): # we don't have names for PK constraints, # it appears we get back None in the pragma for - # FKs also (also it doesn't even appear to be documented on sqlite's docs + # FKs also (also it doesn't even appear to be documented on + # sqlite's docs # at http://www.sqlite.org/pragma.html#pragma_foreign_key_list # how did we ever know that's the "name" field ??) @@ -1018,6 +1057,7 @@ class ReflectFKConstraintTest(fixtures.TestBase): class SavepointTest(fixtures.TablesTest): + """test that savepoints work when we use the correct event setup""" __only_on__ = 'sqlite' @@ -1081,7 +1121,7 @@ class SavepointTest(fixtures.TablesTest): connection = self.bind.connect() transaction = connection.begin() connection.execute(users.insert(), user_id=1, user_name='user1') - trans2 = connection.begin_nested() + connection.begin_nested() connection.execute(users.insert(), user_id=2, user_name='user2') trans3 = connection.begin() connection.execute(users.insert(), user_id=3, user_name='user3') @@ -1169,8 +1209,8 @@ class TypeReflectionTest(fixtures.TestBase): if warnings: def go(): return dialect._resolve_type_affinity(from_) - final_type = testing.assert_warnings(go, - ["Could not instantiate"], regex=True) + final_type = testing.assert_warnings( + go, ["Could not instantiate"], regex=True) else: final_type = dialect._resolve_type_affinity(from_) expected_type = type(to_) @@ -1186,8 +1226,8 @@ class TypeReflectionTest(fixtures.TestBase): if warnings: def go(): return inspector.get_columns("foo")[0] - col_info = testing.assert_warnings(go, - ["Could not instantiate"], regex=True) + col_info = testing.assert_warnings( + go, ["Could not instantiate"], regex=True) else: col_info = inspector.get_columns("foo")[0] expected_type = type(to_) @@ -1207,7 +1247,8 @@ class TypeReflectionTest(fixtures.TestBase): self._test_lookup_direct(self._fixed_lookup_fixture()) def test_lookup_direct_unsupported_args(self): - self._test_lookup_direct(self._unsupported_args_fixture(), warnings=True) + self._test_lookup_direct( + self._unsupported_args_fixture(), warnings=True) def test_lookup_direct_type_affinity(self): self._test_lookup_direct(self._type_affinity_fixture()) @@ -1216,8 +1257,8 @@ class TypeReflectionTest(fixtures.TestBase): self._test_round_trip(self._fixed_lookup_fixture()) def test_round_trip_direct_unsupported_args(self): - self._test_round_trip(self._unsupported_args_fixture(), warnings=True) + self._test_round_trip( + self._unsupported_args_fixture(), warnings=True) def test_round_trip_direct_type_affinity(self): self._test_round_trip(self._type_affinity_fixture()) - -- cgit v1.2.1 From 0ce045bd853ec078943c14fc93b87897d2169882 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 14:46:43 -0500 Subject: - The SQLite dialect, when using the :class:`.sqlite.DATE`, :class:`.sqlite.TIME`, or :class:`.sqlite.DATETIME` types, and given a ``storage_format`` that only renders numbers, will render the types in DDL as ``DATE_CHAR``, ``TIME_CHAR``, and ``DATETIME_CHAR``, so that despite the lack of alpha characters in the values, the column will still deliver the "text affinity". Normally this is not needed, as the textual values within the default storage formats already imply text. fixes #3257 --- doc/build/changelog/changelog_10.rst | 18 ++++++++++ lib/sqlalchemy/dialects/sqlite/base.py | 60 +++++++++++++++++++++++++++++++++- test/dialect/test_sqlite.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index b8b513821..9cc144fc6 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,24 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, sqlite + :tickets: 3257 + + The SQLite dialect, when using the :class:`.sqlite.DATE`, + :class:`.sqlite.TIME`, + or :class:`.sqlite.DATETIME` types, and given a ``storage_format`` that + only renders numbers, will render the types in DDL as + ``DATE_CHAR``, ``TIME_CHAR``, and ``DATETIME_CHAR``, so that despite the + lack of alpha characters in the values, the column will still + deliver the "text affinity". Normally this is not needed, as the + textual values within the default storage formats already + imply text. + + .. seealso:: + + :ref:`sqlite_datetime` + .. change:: :tags: bug, engine :tickets: 3266 diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index 33003297c..ccd7f2539 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -9,6 +9,7 @@ .. dialect:: sqlite :name: SQLite +.. _sqlite_datetime: Date and Time Types ------------------- @@ -23,6 +24,20 @@ These types represent dates and times as ISO formatted strings, which also nicely support ordering. There's no reliance on typical "libc" internals for these functions so historical dates are fully supported. +Ensuring Text affinity +^^^^^^^^^^^^^^^^^^^^^^ + +The DDL rendered for these types is the standard ``DATE``, ``TIME`` +and ``DATETIME`` indicators. However, custom storage formats can also be +applied to these types. When the +storage format is detected as containing no alpha characters, the DDL for +these types is rendered as ``DATE_CHAR``, ``TIME_CHAR``, and ``DATETIME_CHAR``, +so that the column continues to have textual affinity. + +.. seealso:: + + `Type Affinity `_ - in the SQLite documentation + .. _sqlite_autoincrement: SQLite Auto Incrementing Behavior @@ -255,7 +270,7 @@ from ... import util from ...engine import default, reflection from ...sql import compiler -from ...types import (BLOB, BOOLEAN, CHAR, DATE, DECIMAL, FLOAT, +from ...types import (BLOB, BOOLEAN, CHAR, DECIMAL, FLOAT, INTEGER, REAL, NUMERIC, SMALLINT, TEXT, TIMESTAMP, VARCHAR) @@ -271,6 +286,25 @@ class _DateTimeMixin(object): if storage_format is not None: self._storage_format = storage_format + @property + def format_is_text_affinity(self): + """return True if the storage format will automatically imply + a TEXT affinity. + + If the storage format contains no non-numeric characters, + it will imply a NUMERIC storage format on SQLite; in this case, + the type will generate its DDL as DATE_CHAR, DATETIME_CHAR, + TIME_CHAR. + + .. versionadded:: 1.0.0 + + """ + spec = self._storage_format % { + "year": 0, "month": 0, "day": 0, "hour": 0, + "minute": 0, "second": 0, "microsecond": 0 + } + return bool(re.search(r'[^0-9]', spec)) + def adapt(self, cls, **kw): if issubclass(cls, _DateTimeMixin): if self._storage_format: @@ -526,7 +560,9 @@ ischema_names = { 'BOOLEAN': sqltypes.BOOLEAN, 'CHAR': sqltypes.CHAR, 'DATE': sqltypes.DATE, + 'DATE_CHAR': sqltypes.DATE, 'DATETIME': sqltypes.DATETIME, + 'DATETIME_CHAR': sqltypes.DATETIME, 'DOUBLE': sqltypes.FLOAT, 'DECIMAL': sqltypes.DECIMAL, 'FLOAT': sqltypes.FLOAT, @@ -537,6 +573,7 @@ ischema_names = { 'SMALLINT': sqltypes.SMALLINT, 'TEXT': sqltypes.TEXT, 'TIME': sqltypes.TIME, + 'TIME_CHAR': sqltypes.TIME, 'TIMESTAMP': sqltypes.TIMESTAMP, 'VARCHAR': sqltypes.VARCHAR, 'NVARCHAR': sqltypes.NVARCHAR, @@ -670,6 +707,27 @@ class SQLiteTypeCompiler(compiler.GenericTypeCompiler): def visit_large_binary(self, type_): return self.visit_BLOB(type_) + def visit_DATETIME(self, type_): + if not isinstance(type_, _DateTimeMixin) or \ + type_.format_is_text_affinity: + return super(SQLiteTypeCompiler, self).visit_DATETIME(type_) + else: + return "DATETIME_CHAR" + + def visit_DATE(self, type_): + if not isinstance(type_, _DateTimeMixin) or \ + type_.format_is_text_affinity: + return super(SQLiteTypeCompiler, self).visit_DATE(type_) + else: + return "DATE_CHAR" + + def visit_TIME(self, type_): + if not isinstance(type_, _DateTimeMixin) or \ + type_.format_is_text_affinity: + return super(SQLiteTypeCompiler, self).visit_TIME(type_) + else: + return "TIME_CHAR" + class SQLiteIdentifierPreparer(compiler.IdentifierPreparer): reserved_words = set([ diff --git a/test/dialect/test_sqlite.py b/test/dialect/test_sqlite.py index 04e82e686..22772d2fb 100644 --- a/test/dialect/test_sqlite.py +++ b/test/dialect/test_sqlite.py @@ -128,6 +128,53 @@ class TestTypes(fixtures.TestBase, AssertsExecutionResults): (datetime.datetime(2010, 10, 15, 12, 37),)] ) + @testing.provide_metadata + def test_custom_datetime_text_affinity(self): + sqlite_date = sqlite.DATETIME( + storage_format="%(year)04d%(month)02d%(day)02d" + "%(hour)02d%(minute)02d%(second)02d", + regexp=r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})", + ) + t = Table('t', self.metadata, Column('d', sqlite_date)) + self.metadata.create_all(testing.db) + testing.db.execute( + t.insert(). + values(d=datetime.datetime(2010, 10, 15, 12, 37, 0))) + testing.db.execute("insert into t (d) values ('20040521000000')") + eq_( + testing.db.execute("select * from t order by d").fetchall(), + [('20040521000000',), ('20101015123700',)] + ) + eq_( + testing.db.execute(select([t.c.d]).order_by(t.c.d)).fetchall(), + [ + (datetime.datetime(2004, 5, 21, 0, 0),), + (datetime.datetime(2010, 10, 15, 12, 37),)] + ) + + @testing.provide_metadata + def test_custom_date_text_affinity(self): + sqlite_date = sqlite.DATE( + storage_format="%(year)04d%(month)02d%(day)02d", + regexp=r"(\d{4})(\d{2})(\d{2})", + ) + t = Table('t', self.metadata, Column('d', sqlite_date)) + self.metadata.create_all(testing.db) + testing.db.execute( + t.insert(). + values(d=datetime.date(2010, 10, 15))) + testing.db.execute("insert into t (d) values ('20040521')") + eq_( + testing.db.execute("select * from t order by d").fetchall(), + [('20040521',), ('20101015',)] + ) + eq_( + testing.db.execute(select([t.c.d]).order_by(t.c.d)).fetchall(), + [ + (datetime.date(2004, 5, 21),), + (datetime.date(2010, 10, 15),)] + ) + @testing.provide_metadata def test_custom_date(self): sqlite_date = sqlite.DATE( @@ -1167,6 +1214,16 @@ class TypeReflectionTest(fixtures.TestBase): (sqltypes.Time, sqltypes.TIME()), (sqltypes.BOOLEAN, sqltypes.BOOLEAN()), (sqltypes.Boolean, sqltypes.BOOLEAN()), + (sqlite.DATE( + storage_format="%(year)04d%(month)02d%(day)02d", + ), sqltypes.DATE()), + (sqlite.TIME( + storage_format="%(hour)02d%(minute)02d%(second)02d", + ), sqltypes.TIME()), + (sqlite.DATETIME( + storage_format="%(year)04d%(month)02d%(day)02d" + "%(hour)02d%(minute)02d%(second)02d", + ), sqltypes.DATETIME()), ] def _unsupported_args_fixture(self): -- cgit v1.2.1 From 0639c199a547343d62134d2f233225fd2862ec45 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 16:34:43 -0500 Subject: - move inner calls to _revalidate_connection() outside of existing _handle_dbapi_error(); these are now handled already and the reentrant call is not needed / breaks things. Adjustment to 41e7253dee168b8c26c49 / --- lib/sqlalchemy/engine/base.py | 17 +++++++++-------- test/engine/test_parseconnect.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 901ab07eb..235e1bf43 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -814,11 +814,11 @@ class Connection(Connectable): fn(self, default, multiparams, params) try: - try: - conn = self.__connection - except AttributeError: - conn = self._revalidate_connection() + conn = self.__connection + except AttributeError: + conn = self._revalidate_connection() + try: dialect = self.dialect ctx = dialect.execution_ctx_cls._init_default( dialect, self, conn) @@ -952,11 +952,11 @@ class Connection(Connectable): a :class:`.ResultProxy`.""" try: - try: - conn = self.__connection - except AttributeError: - conn = self._revalidate_connection() + conn = self.__connection + except AttributeError: + conn = self._revalidate_connection() + try: context = constructor(dialect, self, conn, *args) except Exception as e: self._handle_dbapi_exception(e, @@ -1246,6 +1246,7 @@ class Connection(Connectable): @classmethod def _handle_dbapi_exception_noconnection( cls, e, dialect, engine, connection): + exc_info = sys.exc_info() is_disconnect = dialect.is_disconnect(e, None, None) diff --git a/test/engine/test_parseconnect.py b/test/engine/test_parseconnect.py index 72a089aca..b6d08ceba 100644 --- a/test/engine/test_parseconnect.py +++ b/test/engine/test_parseconnect.py @@ -7,6 +7,7 @@ from sqlalchemy.testing import fixtures from sqlalchemy import testing from sqlalchemy.testing.mock import Mock, MagicMock from sqlalchemy import event +from sqlalchemy import select dialect = None @@ -279,7 +280,7 @@ class CreateEngineTest(fixtures.TestBase): ) @testing.requires.sqlite - def test_handle_error_event_reconnect(self): + def test_handle_error_event_revalidate(self): e = create_engine('sqlite://') dbapi = MockDBAPI() sqlite3 = e.dialect.dbapi @@ -295,6 +296,7 @@ class CreateEngineTest(fixtures.TestBase): def handle_error(ctx): assert ctx.engine is eng assert ctx.connection is conn + assert isinstance(ctx.sqlalchemy_exception, exc.ProgrammingError) raise MySpecialException("failed operation") conn = eng.connect() @@ -308,6 +310,37 @@ class CreateEngineTest(fixtures.TestBase): conn._revalidate_connection ) + @testing.requires.sqlite + def test_handle_error_event_implicit_revalidate(self): + e = create_engine('sqlite://') + dbapi = MockDBAPI() + sqlite3 = e.dialect.dbapi + dbapi.Error = sqlite3.Error, + dbapi.ProgrammingError = sqlite3.ProgrammingError + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi, _initialize=False) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is conn + assert isinstance(ctx.sqlalchemy_exception, exc.ProgrammingError) + raise MySpecialException("failed operation") + + conn = eng.connect() + conn.invalidate() + + dbapi.connect = Mock( + side_effect=sqlite3.ProgrammingError("random error")) + + assert_raises( + MySpecialException, + conn.execute, select([1]) + ) + @testing.requires.sqlite def test_handle_error_custom_connect(self): e = create_engine('sqlite://') -- cgit v1.2.1 From b8114a357684ab3232ff90ceb0da16dad080d1ac Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 5 Dec 2014 19:08:47 -0500 Subject: - adjust _revalidate_connection() again such that we pass a _wrap=False to it, so that we say we will do the wrapping just once right here in _execute_context() / _execute_default(). An adjustment is made to _handle_dbapi_error() to not assume self.__connection in case we are already in an invalidated state further adjustment to 0639c199a547343d62134d2f233225fd2862ec45, 41e7253dee168b8c26c49, #3266 --- lib/sqlalchemy/engine/base.py | 46 ++++++++++++++++++++---------------- lib/sqlalchemy/engine/threadlocal.py | 5 +++- test/engine/test_parseconnect.py | 2 +- test/engine/test_reconnect.py | 4 ++-- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 235e1bf43..23348469d 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -265,18 +265,18 @@ class Connection(Connectable): try: return self.__connection except AttributeError: - return self._revalidate_connection() + return self._revalidate_connection(_wrap=True) - def _revalidate_connection(self): + def _revalidate_connection(self, _wrap): if self.__branch_from: - return self.__branch_from._revalidate_connection() - + return self.__branch_from._revalidate_connection(_wrap=_wrap) if self.__can_reconnect and self.__invalid: if self.__transaction is not None: raise exc.InvalidRequestError( "Can't reconnect until invalid " "transaction is rolled back") - self.__connection = self.engine.raw_connection(self) + self.__connection = self.engine.raw_connection( + _connection=self, _wrap=_wrap) self.__invalid = False return self.__connection raise exc.ResourceClosedError("This Connection is closed") @@ -814,11 +814,11 @@ class Connection(Connectable): fn(self, default, multiparams, params) try: - conn = self.__connection - except AttributeError: - conn = self._revalidate_connection() + try: + conn = self.__connection + except AttributeError: + conn = self._revalidate_connection(_wrap=False) - try: dialect = self.dialect ctx = dialect.execution_ctx_cls._init_default( dialect, self, conn) @@ -952,16 +952,17 @@ class Connection(Connectable): a :class:`.ResultProxy`.""" try: - conn = self.__connection - except AttributeError: - conn = self._revalidate_connection() + try: + conn = self.__connection + except AttributeError: + conn = self._revalidate_connection(_wrap=False) - try: context = constructor(dialect, self, conn, *args) except Exception as e: - self._handle_dbapi_exception(e, - util.text_type(statement), parameters, - None, None) + self._handle_dbapi_exception( + e, + util.text_type(statement), parameters, + None, None) if context.compiled: context.pre_exec() @@ -1149,7 +1150,10 @@ class Connection(Connectable): self._is_disconnect = \ isinstance(e, self.dialect.dbapi.Error) and \ not self.closed and \ - self.dialect.is_disconnect(e, self.__connection, cursor) + self.dialect.is_disconnect( + e, + self.__connection if not self.invalidated else None, + cursor) if context: context.is_disconnect = self._is_disconnect @@ -1953,7 +1957,9 @@ class Engine(Connectable, log.Identified): """ return self.run_callable(self.dialect.has_table, table_name, schema) - def _wrap_pool_connect(self, fn, connection=None): + def _wrap_pool_connect(self, fn, connection, wrap=True): + if not wrap: + return fn() dialect = self.dialect try: return fn() @@ -1961,7 +1967,7 @@ class Engine(Connectable, log.Identified): Connection._handle_dbapi_exception_noconnection( e, dialect, self, connection) - def raw_connection(self, _connection=None): + def raw_connection(self, _connection=None, _wrap=True): """Return a "raw" DBAPI connection from the connection pool. The returned object is a proxied version of the DBAPI @@ -1978,7 +1984,7 @@ class Engine(Connectable, log.Identified): """ return self._wrap_pool_connect( - self.pool.unique_connection, _connection) + self.pool.unique_connection, _connection, _wrap) class OptionEngine(Engine): diff --git a/lib/sqlalchemy/engine/threadlocal.py b/lib/sqlalchemy/engine/threadlocal.py index 71caac626..824b68fdf 100644 --- a/lib/sqlalchemy/engine/threadlocal.py +++ b/lib/sqlalchemy/engine/threadlocal.py @@ -59,7 +59,10 @@ class TLEngine(base.Engine): # guards against pool-level reapers, if desired. # or not connection.connection.is_valid: connection = self._tl_connection_cls( - self, self._wrap_pool_connect(self.pool.connect), **kw) + self, + self._wrap_pool_connect( + self.pool.connect, connection, wrap=True), + **kw) self._connections.conn = weakref.ref(connection) return connection._increment_connect() diff --git a/test/engine/test_parseconnect.py b/test/engine/test_parseconnect.py index b6d08ceba..4a3da7d1c 100644 --- a/test/engine/test_parseconnect.py +++ b/test/engine/test_parseconnect.py @@ -307,7 +307,7 @@ class CreateEngineTest(fixtures.TestBase): assert_raises( MySpecialException, - conn._revalidate_connection + getattr, conn, 'connection' ) @testing.requires.sqlite diff --git a/test/engine/test_reconnect.py b/test/engine/test_reconnect.py index 4500ada6a..0efce87ce 100644 --- a/test/engine/test_reconnect.py +++ b/test/engine/test_reconnect.py @@ -517,7 +517,7 @@ class RealReconnectTest(fixtures.TestBase): assert c1.invalidated assert c1_branch.invalidated - c1_branch._revalidate_connection() + c1_branch._revalidate_connection(_wrap=True) assert not c1.invalidated assert not c1_branch.invalidated @@ -535,7 +535,7 @@ class RealReconnectTest(fixtures.TestBase): assert c1.invalidated assert c1_branch.invalidated - c1._revalidate_connection() + c1._revalidate_connection(_wrap=True) assert not c1.invalidated assert not c1_branch.invalidated -- cgit v1.2.1 From 95cd2003bbe1b5da2d3c78ac845855126e03de2f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 6 Dec 2014 12:39:18 -0500 Subject: pep8 --- test/dialect/mssql/test_types.py | 247 ++++++++++++++++++++++----------------- 1 file changed, 141 insertions(+), 106 deletions(-) diff --git a/test/dialect/mssql/test_types.py b/test/dialect/mssql/test_types.py index 9dc1983ae..24f0eb0be 100644 --- a/test/dialect/mssql/test_types.py +++ b/test/dialect/mssql/test_types.py @@ -2,12 +2,14 @@ from sqlalchemy.testing import eq_, engines, pickleable import datetime import os -from sqlalchemy import * +from sqlalchemy import Table, Column, MetaData, Float, \ + Integer, String, Boolean, TIMESTAMP, Sequence, Numeric, select, \ + Date, Time, DateTime, DefaultClause, PickleType, text from sqlalchemy import types, schema from sqlalchemy.databases import mssql from sqlalchemy.dialects.mssql.base import TIME from sqlalchemy.testing import fixtures, \ - AssertsExecutionResults, ComparesTables + AssertsExecutionResults, ComparesTables from sqlalchemy import testing from sqlalchemy.testing import emits_warning_on import decimal @@ -32,6 +34,7 @@ class TimeTypeTest(fixtures.TestBase): class TypeDDLTest(fixtures.TestBase): + def test_boolean(self): "Exercise type specification for boolean type." @@ -39,7 +42,7 @@ class TypeDDLTest(fixtures.TestBase): # column type, args, kwargs, expected ddl (Boolean, [], {}, 'BIT'), - ] + ] metadata = MetaData() table_args = ['test_mssql_boolean', metadata] @@ -54,11 +57,11 @@ class TypeDDLTest(fixtures.TestBase): for col in boolean_table.c: index = int(col.name[1:]) - testing.eq_(gen.get_column_specification(col), - "%s %s" % (col.name, columns[index][3])) + testing.eq_( + gen.get_column_specification(col), + "%s %s" % (col.name, columns[index][3])) self.assert_(repr(col)) - def test_numeric(self): "Exercise type specification and options for numeric types." @@ -88,7 +91,7 @@ class TypeDDLTest(fixtures.TestBase): 'TINYINT'), (types.SmallInteger, [], {}, 'SMALLINT'), - ] + ] metadata = MetaData() table_args = ['test_mssql_numeric', metadata] @@ -103,11 +106,11 @@ class TypeDDLTest(fixtures.TestBase): for col in numeric_table.c: index = int(col.name[1:]) - testing.eq_(gen.get_column_specification(col), - "%s %s" % (col.name, columns[index][3])) + testing.eq_( + gen.get_column_specification(col), + "%s %s" % (col.name, columns[index][3])) self.assert_(repr(col)) - def test_char(self): """Exercise COLLATE-ish options on string types.""" @@ -149,7 +152,7 @@ class TypeDDLTest(fixtures.TestBase): 'NTEXT'), (mssql.MSNText, [], {'collation': 'Latin1_General_CI_AS'}, 'NTEXT COLLATE Latin1_General_CI_AS'), - ] + ] metadata = MetaData() table_args = ['test_mssql_charset', metadata] @@ -164,11 +167,11 @@ class TypeDDLTest(fixtures.TestBase): for col in charset_table.c: index = int(col.name[1:]) - testing.eq_(gen.get_column_specification(col), - "%s %s" % (col.name, columns[index][3])) + testing.eq_( + gen.get_column_specification(col), + "%s %s" % (col.name, columns[index][3])) self.assert_(repr(col)) - def test_timestamp(self): """Exercise TIMESTAMP column.""" @@ -176,9 +179,10 @@ class TypeDDLTest(fixtures.TestBase): metadata = MetaData() spec, expected = (TIMESTAMP, 'TIMESTAMP') - t = Table('mssql_ts', metadata, - Column('id', Integer, primary_key=True), - Column('t', spec, nullable=None)) + t = Table( + 'mssql_ts', metadata, + Column('id', Integer, primary_key=True), + Column('t', spec, nullable=None)) gen = dialect.ddl_compiler(dialect, schema.CreateTable(t)) testing.eq_(gen.get_column_specification(t.c.t), "t %s" % expected) self.assert_(repr(t.c.t)) @@ -255,7 +259,11 @@ class TypeDDLTest(fixtures.TestBase): % (col.name, columns[index][3])) self.assert_(repr(col)) -class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTables): +metadata = None + + +class TypeRoundTripTest( + fixtures.TestBase, AssertsExecutionResults, ComparesTables): __only_on__ = 'mssql' @classmethod @@ -266,15 +274,18 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl def teardown(self): metadata.drop_all() - @testing.fails_on_everything_except('mssql+pyodbc', - 'this is some pyodbc-specific feature') + @testing.fails_on_everything_except( + 'mssql+pyodbc', + 'this is some pyodbc-specific feature') def test_decimal_notation(self): - numeric_table = Table('numeric_table', metadata, Column('id', - Integer, Sequence('numeric_id_seq', - optional=True), primary_key=True), - Column('numericcol', - Numeric(precision=38, scale=20, - asdecimal=True))) + numeric_table = Table( + 'numeric_table', metadata, + Column( + 'id', Integer, + Sequence('numeric_id_seq', optional=True), primary_key=True), + Column( + 'numericcol', + Numeric(precision=38, scale=20, asdecimal=True))) metadata.create_all() test_items = [decimal.Decimal(d) for d in ( '1500000.00000000000000000000', @@ -323,7 +334,7 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl '000000000000.32E12', '00000000000000.1E+12', '000000000000.2E-32', - )] + )] for value in test_items: numeric_table.insert().execute(numericcol=value) @@ -332,10 +343,13 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl assert value[0] in test_items, "%r not in test_items" % value[0] def test_float(self): - float_table = Table('float_table', metadata, Column('id', - Integer, Sequence('numeric_id_seq', - optional=True), primary_key=True), - Column('floatcol', Float())) + float_table = Table( + 'float_table', metadata, + Column( + 'id', Integer, + Sequence('numeric_id_seq', optional=True), primary_key=True), + Column('floatcol', Float())) + metadata.create_all() try: test_items = [float(d) for d in ( @@ -363,13 +377,12 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl '1E-6', '1E-7', '1E-8', - )] + )] for value in test_items: float_table.insert().execute(floatcol=value) except Exception as e: raise e - # todo this should suppress warnings, but it does not @emits_warning_on('mssql+mxodbc', r'.*does not have any indexes.*') def test_dates(self): @@ -417,20 +430,20 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl (mssql.MSDateTime2, [1], {}, 'DATETIME2(1)', ['>=', (10,)]), - ] + ] table_args = ['test_mssql_dates', metadata] for index, spec in enumerate(columns): type_, args, kw, res, requires = spec[0:5] - if requires and testing._is_excluded('mssql', *requires) \ - or not requires: - c = Column('c%s' % index, type_(*args, - **kw), nullable=None) + if requires and \ + testing._is_excluded('mssql', *requires) or not requires: + c = Column('c%s' % index, type_(*args, **kw), nullable=None) testing.db.dialect.type_descriptor(c.type) table_args.append(c) dates_table = Table(*table_args) - gen = testing.db.dialect.ddl_compiler(testing.db.dialect, - schema.CreateTable(dates_table)) + gen = testing.db.dialect.ddl_compiler( + testing.db.dialect, + schema.CreateTable(dates_table)) for col in dates_table.c: index = int(col.name[1:]) testing.eq_(gen.get_column_specification(col), '%s %s' @@ -443,13 +456,14 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl self.assert_types_base(col, dates_table.c[col.key]) def test_date_roundtrip(self): - t = Table('test_dates', metadata, - Column('id', Integer, - Sequence('datetest_id_seq', optional=True), - primary_key=True), - Column('adate', Date), - Column('atime', Time), - Column('adatetime', DateTime)) + t = Table( + 'test_dates', metadata, + Column('id', Integer, + Sequence('datetest_id_seq', optional=True), + primary_key=True), + Column('adate', Date), + Column('atime', Time), + Column('adatetime', DateTime)) metadata.create_all() d1 = datetime.date(2007, 10, 30) t1 = datetime.time(11, 2, 32) @@ -527,48 +541,57 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl testing.eq_(col.type.length, binary_table.c[col.name].type.length) - def test_autoincrement(self): - Table('ai_1', metadata, - Column('int_y', Integer, primary_key=True), - Column('int_n', Integer, DefaultClause('0'), - primary_key=True, autoincrement=False)) - Table('ai_2', metadata, - Column('int_y', Integer, primary_key=True), - Column('int_n', Integer, DefaultClause('0'), - primary_key=True, autoincrement=False)) - Table('ai_3', metadata, - Column('int_n', Integer, DefaultClause('0'), - primary_key=True, autoincrement=False), - Column('int_y', Integer, primary_key=True)) - Table('ai_4', metadata, - Column('int_n', Integer, DefaultClause('0'), - primary_key=True, autoincrement=False), - Column('int_n2', Integer, DefaultClause('0'), - primary_key=True, autoincrement=False)) - Table('ai_5', metadata, - Column('int_y', Integer, primary_key=True), - Column('int_n', Integer, DefaultClause('0'), - primary_key=True, autoincrement=False)) - Table('ai_6', metadata, - Column('o1', String(1), DefaultClause('x'), - primary_key=True), - Column('int_y', Integer, primary_key=True)) - Table('ai_7', metadata, - Column('o1', String(1), DefaultClause('x'), - primary_key=True), - Column('o2', String(1), DefaultClause('x'), - primary_key=True), - Column('int_y', Integer, primary_key=True)) - Table('ai_8', metadata, - Column('o1', String(1), DefaultClause('x'), - primary_key=True), - Column('o2', String(1), DefaultClause('x'), - primary_key=True)) + Table( + 'ai_1', metadata, + Column('int_y', Integer, primary_key=True), + Column( + 'int_n', Integer, DefaultClause('0'), + primary_key=True, autoincrement=False)) + Table( + 'ai_2', metadata, + Column('int_y', Integer, primary_key=True), + Column('int_n', Integer, DefaultClause('0'), + primary_key=True, autoincrement=False)) + Table( + 'ai_3', metadata, + Column('int_n', Integer, DefaultClause('0'), + primary_key=True, autoincrement=False), + Column('int_y', Integer, primary_key=True)) + + Table( + 'ai_4', metadata, + Column('int_n', Integer, DefaultClause('0'), + primary_key=True, autoincrement=False), + Column('int_n2', Integer, DefaultClause('0'), + primary_key=True, autoincrement=False)) + Table( + 'ai_5', metadata, + Column('int_y', Integer, primary_key=True), + Column('int_n', Integer, DefaultClause('0'), + primary_key=True, autoincrement=False)) + Table( + 'ai_6', metadata, + Column('o1', String(1), DefaultClause('x'), + primary_key=True), + Column('int_y', Integer, primary_key=True)) + Table( + 'ai_7', metadata, + Column('o1', String(1), DefaultClause('x'), + primary_key=True), + Column('o2', String(1), DefaultClause('x'), + primary_key=True), + Column('int_y', Integer, primary_key=True)) + Table( + 'ai_8', metadata, + Column('o1', String(1), DefaultClause('x'), + primary_key=True), + Column('o2', String(1), DefaultClause('x'), + primary_key=True)) metadata.create_all() table_names = ['ai_1', 'ai_2', 'ai_3', 'ai_4', - 'ai_5', 'ai_6', 'ai_7', 'ai_8'] + 'ai_5', 'ai_6', 'ai_7', 'ai_8'] mr = MetaData(testing.db) for name in table_names: @@ -586,27 +609,29 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults, ComparesTabl if testing.db.driver == 'mxodbc': eng = \ - [engines.testing_engine(options={'implicit_returning' - : True})] + [engines.testing_engine(options={ + 'implicit_returning': True})] else: eng = \ - [engines.testing_engine(options={'implicit_returning' - : False}), - engines.testing_engine(options={'implicit_returning' - : True})] + [engines.testing_engine(options={ + 'implicit_returning': False}), + engines.testing_engine(options={ + 'implicit_returning': True})] for counter, engine in enumerate(eng): engine.execute(tbl.insert()) if 'int_y' in tbl.c: assert engine.scalar(select([tbl.c.int_y])) \ == counter + 1 - assert list(engine.execute(tbl.select()).first()).\ - count(counter + 1) == 1 + assert list( + engine.execute(tbl.select()).first()).\ + count(counter + 1) == 1 else: assert 1 \ not in list(engine.execute(tbl.select()).first()) engine.execute(tbl.delete()) + class MonkeyPatchedBinaryTest(fixtures.TestBase): __only_on__ = 'mssql+pymssql' @@ -622,7 +647,12 @@ class MonkeyPatchedBinaryTest(fixtures.TestBase): result = module.Binary(input) eq_(result, expected_result) +binary_table = None +MyPickleType = None + + class BinaryTest(fixtures.TestBase, AssertsExecutionResults): + """Test the Binary and VarBinary types""" __only_on__ = 'mssql' @@ -655,7 +685,7 @@ class BinaryTest(fixtures.TestBase, AssertsExecutionResults): Column('misc', String(30)), Column('pickled', PickleType), Column('mypickle', MyPickleType), - ) + ) binary_table.create() def teardown(self): @@ -679,7 +709,7 @@ class BinaryTest(fixtures.TestBase, AssertsExecutionResults): data_slice=stream1[0:100], pickled=testobj1, mypickle=testobj3, - ) + ) binary_table.insert().execute( primary_id=2, misc='binary_data_two.dat', @@ -687,7 +717,7 @@ class BinaryTest(fixtures.TestBase, AssertsExecutionResults): data_image=stream2, data_slice=stream2[0:99], pickled=testobj2, - ) + ) # TODO: pyodbc does not seem to accept "None" for a VARBINARY # column (data=None). error: [Microsoft][ODBC SQL Server @@ -697,17 +727,21 @@ class BinaryTest(fixtures.TestBase, AssertsExecutionResults): # misc='binary_data_two.dat', data=None, data_image=None, # data_slice=stream2[0:99], pickled=None) - binary_table.insert().execute(primary_id=3, - misc='binary_data_two.dat', data_image=None, - data_slice=stream2[0:99], pickled=None) + binary_table.insert().execute( + primary_id=3, + misc='binary_data_two.dat', data_image=None, + data_slice=stream2[0:99], pickled=None) for stmt in \ binary_table.select(order_by=binary_table.c.primary_id), \ - text('select * from binary_table order by ' - 'binary_table.primary_id', - typemap=dict(data=mssql.MSVarBinary(8000), - data_image=mssql.MSImage, - data_slice=types.BINARY(100), pickled=PickleType, - mypickle=MyPickleType), bind=testing.db): + text( + 'select * from binary_table order by ' + 'binary_table.primary_id', + typemap=dict( + data=mssql.MSVarBinary(8000), + data_image=mssql.MSImage, + data_slice=types.BINARY(100), pickled=PickleType, + mypickle=MyPickleType), + bind=testing.db): l = stmt.execute().fetchall() eq_(list(stream1), list(l[0]['data'])) paddedstream = list(stream1[0:100]) @@ -721,7 +755,8 @@ class BinaryTest(fixtures.TestBase, AssertsExecutionResults): eq_(l[0]['mypickle'].stuff, 'this is the right stuff') def load_stream(self, name, len=3000): - fp = open(os.path.join(os.path.dirname(__file__), "..", "..", name), 'rb') + fp = open( + os.path.join(os.path.dirname(__file__), "..", "..", name), 'rb') stream = fp.read(len) fp.close() return stream -- cgit v1.2.1 From c24423bc2e3fd227bf4a86599e28407bd190ee9e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 6 Dec 2014 13:29:32 -0500 Subject: - enhance only_on() to work with compound specs - fix "temporary_tables" requirement --- lib/sqlalchemy/testing/exclusions.py | 2 +- lib/sqlalchemy/testing/requirements.py | 5 +++++ test/requirements.py | 12 +++++++++--- test/sql/test_metadata.py | 3 ++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/sqlalchemy/testing/exclusions.py b/lib/sqlalchemy/testing/exclusions.py index f94724608..0aff43ae1 100644 --- a/lib/sqlalchemy/testing/exclusions.py +++ b/lib/sqlalchemy/testing/exclusions.py @@ -425,7 +425,7 @@ def skip(db, reason=None): def only_on(dbs, reason=None): return only_if( - OrPredicate([SpecPredicate(db) for db in util.to_list(dbs)]) + OrPredicate([Predicate.as_predicate(db) for db in util.to_list(dbs)]) ) diff --git a/lib/sqlalchemy/testing/requirements.py b/lib/sqlalchemy/testing/requirements.py index da3e3128a..5744431cb 100644 --- a/lib/sqlalchemy/testing/requirements.py +++ b/lib/sqlalchemy/testing/requirements.py @@ -322,6 +322,11 @@ class SuiteRequirements(Requirements): """target dialect supports listing of temporary table names""" return exclusions.closed() + @property + def temporary_tables(self): + """target database supports temporary tables""" + return exclusions.open() + @property def temporary_views(self): """target database supports temporary views""" diff --git a/test/requirements.py b/test/requirements.py index d1b7913f0..22ac13fe8 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -127,9 +127,15 @@ class DefaultRequirements(SuiteRequirements): ) @property - def temporary_table(self): - """Target database must support CREATE TEMPORARY TABLE""" - return exclusions.open() + def temporary_tables(self): + """target database supports temporary tables""" + return skip_if( + ["mssql"], "sql server has some other syntax?" + ) + + @property + def temp_table_reflection(self): + return self.temporary_tables @property def reflectable_autoincrement(self): diff --git a/test/sql/test_metadata.py b/test/sql/test_metadata.py index 0aa5d7305..52ecf88c5 100644 --- a/test/sql/test_metadata.py +++ b/test/sql/test_metadata.py @@ -1160,9 +1160,10 @@ class InfoTest(fixtures.TestBase): t = Table('x', MetaData(), info={'foo': 'bar'}) eq_(t.info, {'foo': 'bar'}) + class TableTest(fixtures.TestBase, AssertsCompiledSQL): - @testing.requires.temporary_table + @testing.requires.temporary_tables @testing.skip_if('mssql', 'different col format') def test_prefixes(self): from sqlalchemy import Table -- cgit v1.2.1 From c8817e608788799837a91b1d2616227594698d2b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 6 Dec 2014 13:30:51 -0500 Subject: - SQL Server 2012 now recommends VARCHAR(max), NVARCHAR(max), VARBINARY(max) for large text/binary types. The MSSQL dialect will now respect this based on version detection, as well as the new ``deprecate_large_types`` flag. fixes #3039 --- doc/build/changelog/changelog_10.rst | 13 +++++ doc/build/changelog/migration_10.rst | 8 +++ lib/sqlalchemy/dialects/mssql/base.py | 105 +++++++++++++++++++++++++++++++--- lib/sqlalchemy/sql/sqltypes.py | 2 +- test/dialect/mssql/test_engine.py | 3 +- test/dialect/mssql/test_reflection.py | 11 +++- test/dialect/mssql/test_types.py | 71 ++++++++++++++++++++--- test/requirements.py | 12 ++++ 8 files changed, 201 insertions(+), 24 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 9cc144fc6..6d99095d9 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,19 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, mssql + :tickets: 3039 + + SQL Server 2012 now recommends VARCHAR(max), NVARCHAR(max), + VARBINARY(max) for large text/binary types. The MSSQL dialect will + now respect this based on version detection, as well as the new + ``deprecate_large_types`` flag. + + .. seealso:: + + :ref:`mssql_large_type_deprecation` + .. change:: :tags: bug, sqlite :tickets: 3257 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 15e066a75..562bb9f1b 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1619,6 +1619,14 @@ when using ODBC to avoid this issue entirely. :ticket:`3182` +SQL Server 2012 large text / binary types render as VARCHAR, NVARCHAR, VARBINARY +-------------------------------------------------------------------------------- + +The rendering of the :class:`.Text`, :class:`.UnicodeText`, and :class:`.LargeBinary` +types has been changed for SQL Server 2012 and greater, with options +to control the behavior completely, based on deprecation guidelines from +Microsoft. See :ref:`mssql_large_type_deprecation` for details. + .. _change_3204: SQLite/Oracle have distinct methods for temporary table/view name reporting diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index dad02ee0f..5d84975c0 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -226,6 +226,53 @@ The DATE and TIME types are not available for MSSQL 2005 and previous - if a server version below 2008 is detected, DDL for these types will be issued as DATETIME. +.. _mssql_large_type_deprecation: + +Large Text/Binary Type Deprecation +---------------------------------- + +Per `SQL Server 2012/2014 Documentation `_, +the ``NTEXT``, ``TEXT`` and ``IMAGE`` datatypes are to be removed from SQL Server +in a future release. SQLAlchemy normally relates these types to the +:class:`.UnicodeText`, :class:`.Text` and :class:`.LargeBinary` datatypes. + +In order to accommodate this change, a new flag ``deprecate_large_types`` +is added to the dialect, which will be automatically set based on detection +of the server version in use, if not otherwise set by the user. The +behavior of this flag is as follows: + +* When this flag is ``True``, the :class:`.UnicodeText`, :class:`.Text` and + :class:`.LargeBinary` datatypes, when used to render DDL, will render the + types ``NVARCHAR(max)``, ``VARCHAR(max)``, and ``VARBINARY(max)``, + respectively. This is a new behavior as of the addition of this flag. + +* When this flag is ``False``, the :class:`.UnicodeText`, :class:`.Text` and + :class:`.LargeBinary` datatypes, when used to render DDL, will render the + types ``NTEXT``, ``TEXT``, and ``IMAGE``, + respectively. This is the long-standing behavior of these types. + +* The flag begins with the value ``None``, before a database connection is + established. If the dialect is used to render DDL without the flag being + set, it is interpreted the same as ``False``. + +* On first connection, the dialect detects if SQL Server version 2012 or greater + is in use; if the flag is still at ``None``, it sets it to ``True`` or + ``False`` based on whether 2012 or greater is detected. + +* The flag can be set to either ``True`` or ``False`` when the dialect + is created, typically via :func:`.create_engine`:: + + eng = create_engine("mssql+pymssql://user:pass@host/db", + deprecate_large_types=True) + +* Complete control over whether the "old" or "new" types are rendered is + available in all SQLAlchemy versions by using the UPPERCASE type objects + instead: :class:`.NVARCHAR`, :class:`.VARCHAR`, :class:`.types.VARBINARY`, + :class:`.TEXT`, :class:`.mssql.NTEXT`, :class:`.mssql.IMAGE` will always remain + fixed and always output exactly that type. + +.. versionadded:: 1.0.0 + .. _mssql_indexes: Clustered Index Support @@ -367,19 +414,20 @@ import operator import re from ... import sql, schema as sa_schema, exc, util -from ...sql import compiler, expression, \ - util as sql_util, cast +from ...sql import compiler, expression, util as sql_util from ... import engine from ...engine import reflection, default from ... import types as sqltypes from ...types import INTEGER, BIGINT, SMALLINT, DECIMAL, NUMERIC, \ FLOAT, TIMESTAMP, DATETIME, DATE, BINARY,\ - VARBINARY, TEXT, VARCHAR, NVARCHAR, CHAR, NCHAR + TEXT, VARCHAR, NVARCHAR, CHAR, NCHAR from ...util import update_wrapper from . import information_schema as ischema +# http://sqlserverbuilds.blogspot.com/ +MS_2012_VERSION = (11,) MS_2008_VERSION = (10,) MS_2005_VERSION = (9,) MS_2000_VERSION = (8,) @@ -545,6 +593,26 @@ class NTEXT(sqltypes.UnicodeText): __visit_name__ = 'NTEXT' +class VARBINARY(sqltypes.VARBINARY, sqltypes.LargeBinary): + """The MSSQL VARBINARY type. + + This type extends both :class:`.types.VARBINARY` and + :class:`.types.LargeBinary`. In "deprecate_large_types" mode, + the :class:`.types.LargeBinary` type will produce ``VARBINARY(max)`` + on SQL Server. + + .. versionadded:: 1.0.0 + + .. seealso:: + + :ref:`mssql_large_type_deprecation` + + + + """ + __visit_name__ = 'VARBINARY' + + class IMAGE(sqltypes.LargeBinary): __visit_name__ = 'IMAGE' @@ -683,8 +751,17 @@ class MSTypeCompiler(compiler.GenericTypeCompiler): def visit_unicode(self, type_): return self.visit_NVARCHAR(type_) + def visit_text(self, type_): + if self.dialect.deprecate_large_types: + return self.visit_VARCHAR(type_) + else: + return self.visit_TEXT(type_) + def visit_unicode_text(self, type_): - return self.visit_NTEXT(type_) + if self.dialect.deprecate_large_types: + return self.visit_NVARCHAR(type_) + else: + return self.visit_NTEXT(type_) def visit_NTEXT(self, type_): return self._extend("NTEXT", type_) @@ -717,7 +794,10 @@ class MSTypeCompiler(compiler.GenericTypeCompiler): return self.visit_TIME(type_) def visit_large_binary(self, type_): - return self.visit_IMAGE(type_) + if self.dialect.deprecate_large_types: + return self.visit_VARBINARY(type_) + else: + return self.visit_IMAGE(type_) def visit_IMAGE(self, type_): return "IMAGE" @@ -1370,13 +1450,15 @@ class MSDialect(default.DefaultDialect): query_timeout=None, use_scope_identity=True, max_identifier_length=None, - schema_name="dbo", **opts): + schema_name="dbo", + deprecate_large_types=None, **opts): self.query_timeout = int(query_timeout or 0) self.schema_name = schema_name self.use_scope_identity = use_scope_identity self.max_identifier_length = int(max_identifier_length or 0) or \ self.max_identifier_length + self.deprecate_large_types = deprecate_large_types super(MSDialect, self).__init__(**opts) def do_savepoint(self, connection, name): @@ -1390,6 +1472,9 @@ class MSDialect(default.DefaultDialect): def initialize(self, connection): super(MSDialect, self).initialize(connection) + self._setup_version_attributes() + + def _setup_version_attributes(self): if self.server_version_info[0] not in list(range(8, 17)): # FreeTDS with version 4.2 seems to report here # a number like "95.10.255". Don't know what @@ -1405,6 +1490,9 @@ class MSDialect(default.DefaultDialect): self.implicit_returning = True if self.server_version_info >= MS_2008_VERSION: self.supports_multivalues_insert = True + if self.deprecate_large_types is None: + self.deprecate_large_types = \ + self.server_version_info >= MS_2012_VERSION def _get_default_schema_name(self, connection): if self.server_version_info < MS_2005_VERSION: @@ -1592,12 +1680,11 @@ class MSDialect(default.DefaultDialect): if coltype in (MSString, MSChar, MSNVarchar, MSNChar, MSText, MSNText, MSBinary, MSVarBinary, sqltypes.LargeBinary): + if charlen == -1: + charlen = 'max' kwargs['length'] = charlen if collation: kwargs['collation'] = collation - if coltype == MSText or \ - (coltype in (MSString, MSNVarchar) and charlen == -1): - kwargs.pop('length') if coltype is None: util.warn( diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 94db1d837..9a2de39b4 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -894,7 +894,7 @@ class LargeBinary(_Binary): :param length: optional, a length for the column for use in DDL statements, for those BLOB types that accept a length - (i.e. MySQL). It does *not* produce a small BINARY/VARBINARY + (i.e. MySQL). It does *not* produce a *lengthed* BINARY/VARBINARY type - use the BINARY/VARBINARY types specifically for those. May be safely omitted if no ``CREATE TABLE`` will be issued. Certain databases may require a diff --git a/test/dialect/mssql/test_engine.py b/test/dialect/mssql/test_engine.py index 4b4780d43..a994b1787 100644 --- a/test/dialect/mssql/test_engine.py +++ b/test/dialect/mssql/test_engine.py @@ -157,8 +157,7 @@ class ParseConnectTest(fixtures.TestBase): eq_(dialect.is_disconnect("not an error", None, None), False) - @testing.only_on(['mssql+pyodbc', 'mssql+pymssql'], - "FreeTDS specific test") + @testing.requires.mssql_freetds def test_bad_freetds_warning(self): engine = engines.testing_engine() diff --git a/test/dialect/mssql/test_reflection.py b/test/dialect/mssql/test_reflection.py index 0ef69f656..bee441586 100644 --- a/test/dialect/mssql/test_reflection.py +++ b/test/dialect/mssql/test_reflection.py @@ -24,14 +24,14 @@ class ReflectionTest(fixtures.TestBase, ComparesTables): Column('user_name', types.VARCHAR(20), nullable=False), Column('test1', types.CHAR(5), nullable=False), Column('test2', types.Float(5), nullable=False), - Column('test3', types.Text), + Column('test3', types.Text('max')), Column('test4', types.Numeric, nullable=False), Column('test5', types.DateTime), Column('parent_user_id', types.Integer, ForeignKey('engine_users.user_id')), Column('test6', types.DateTime, nullable=False), - Column('test7', types.Text), - Column('test8', types.LargeBinary), + Column('test7', types.Text('max')), + Column('test8', types.LargeBinary('max')), Column('test_passivedefault2', types.Integer, server_default='5'), Column('test9', types.BINARY(100)), @@ -204,6 +204,11 @@ class InfoCoerceUnicodeTest(fixtures.TestBase, AssertsCompiledSQL): class ReflectHugeViewTest(fixtures.TestBase): __only_on__ = 'mssql' + # crashes on freetds 0.91, not worth it + __skip_if__ = ( + lambda: testing.requires.mssql_freetds.enabled, + ) + def setup(self): self.col_num = 150 diff --git a/test/dialect/mssql/test_types.py b/test/dialect/mssql/test_types.py index 24f0eb0be..5c9157379 100644 --- a/test/dialect/mssql/test_types.py +++ b/test/dialect/mssql/test_types.py @@ -4,7 +4,8 @@ import datetime import os from sqlalchemy import Table, Column, MetaData, Float, \ Integer, String, Boolean, TIMESTAMP, Sequence, Numeric, select, \ - Date, Time, DateTime, DefaultClause, PickleType, text + Date, Time, DateTime, DefaultClause, PickleType, text, Text, \ + UnicodeText, LargeBinary from sqlalchemy import types, schema from sqlalchemy.databases import mssql from sqlalchemy.dialects.mssql.base import TIME @@ -172,6 +173,44 @@ class TypeDDLTest(fixtures.TestBase): "%s %s" % (col.name, columns[index][3])) self.assert_(repr(col)) + def test_large_type_deprecation(self): + d1 = mssql.dialect(deprecate_large_types=True) + d2 = mssql.dialect(deprecate_large_types=False) + d3 = mssql.dialect() + d3.server_version_info = (11, 0) + d3._setup_version_attributes() + d4 = mssql.dialect() + d4.server_version_info = (10, 0) + d4._setup_version_attributes() + + for dialect in (d1, d3): + eq_( + str(Text().compile(dialect=dialect)), + "VARCHAR(max)" + ) + eq_( + str(UnicodeText().compile(dialect=dialect)), + "NVARCHAR(max)" + ) + eq_( + str(LargeBinary().compile(dialect=dialect)), + "VARBINARY(max)" + ) + + for dialect in (d2, d4): + eq_( + str(Text().compile(dialect=dialect)), + "TEXT" + ) + eq_( + str(UnicodeText().compile(dialect=dialect)), + "NTEXT" + ) + eq_( + str(LargeBinary().compile(dialect=dialect)), + "IMAGE" + ) + def test_timestamp(self): """Exercise TIMESTAMP column.""" @@ -485,18 +524,18 @@ class TypeRoundTripTest( @emits_warning_on('mssql+mxodbc', r'.*does not have any indexes.*') @testing.provide_metadata - def test_binary_reflection(self): + def _test_binary_reflection(self, deprecate_large_types): "Exercise type specification for binary types." columns = [ - # column type, args, kwargs, expected ddl + # column type, args, kwargs, expected ddl from reflected (mssql.MSBinary, [], {}, - 'BINARY'), + 'BINARY(1)'), (mssql.MSBinary, [10], {}, 'BINARY(10)'), (types.BINARY, [], {}, - 'BINARY'), + 'BINARY(1)'), (types.BINARY, [10], {}, 'BINARY(10)'), @@ -517,10 +556,12 @@ class TypeRoundTripTest( 'IMAGE'), (types.LargeBinary, [], {}, - 'IMAGE'), + 'IMAGE' if not deprecate_large_types else 'VARBINARY(max)'), ] metadata = self.metadata + metadata.bind = engines.testing_engine( + options={"deprecate_large_types": deprecate_large_types}) table_args = ['test_mssql_binary', metadata] for index, spec in enumerate(columns): type_, args, kw, res = spec @@ -530,17 +571,29 @@ class TypeRoundTripTest( metadata.create_all() reflected_binary = Table('test_mssql_binary', MetaData(testing.db), autoload=True) - for col in reflected_binary.c: + for col, spec in zip(reflected_binary.c, columns): + eq_( + str(col.type), spec[3], + "column %s %s != %s" % (col.key, str(col.type), spec[3]) + ) c1 = testing.db.dialect.type_descriptor(col.type).__class__ c2 = \ testing.db.dialect.type_descriptor( binary_table.c[col.name].type).__class__ - assert issubclass(c1, c2), '%r is not a subclass of %r' \ - % (c1, c2) + assert issubclass(c1, c2), \ + 'column %s: %r is not a subclass of %r' \ + % (col.key, c1, c2) if binary_table.c[col.name].type.length: testing.eq_(col.type.length, binary_table.c[col.name].type.length) + def test_binary_reflection_legacy_large_types(self): + self._test_binary_reflection(False) + + @testing.only_on('mssql >= 11') + def test_binary_reflection_sql2012_large_types(self): + self._test_binary_reflection(True) + def test_autoincrement(self): Table( 'ai_1', metadata, diff --git a/test/requirements.py b/test/requirements.py index 22ac13fe8..ffbdfba23 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -460,6 +460,7 @@ class DefaultRequirements(SuiteRequirements): ) + @property def emulated_lastrowid(self): """"target dialect retrieves cursor.lastrowid or an equivalent @@ -777,6 +778,17 @@ class DefaultRequirements(SuiteRequirements): "Not supported on MySQL + Windows" ) + @property + def mssql_freetds(self): + return only_on( + LambdaPredicate( + lambda config: ( + (against(config, 'mssql+pyodbc') and + config.db.dialect.freetds) + or against(config, 'mssql+pymssql') + ) + ) + ) @property def selectone(self): -- cgit v1.2.1 From 60e6ac8856e5f7f257e1797280d1510682ae8fb7 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 7 Dec 2014 18:54:52 -0500 Subject: - rework the assert_sql system so that we have a context manager to work with, use events that are local to the engine and to the run and are removed afterwards. --- lib/sqlalchemy/testing/assertions.py | 13 +++-- lib/sqlalchemy/testing/assertsql.py | 92 ++++++++++++++++++++++++++---------- lib/sqlalchemy/testing/engines.py | 3 -- 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/lib/sqlalchemy/testing/assertions.py b/lib/sqlalchemy/testing/assertions.py index bf7c27a89..66d1f3cb0 100644 --- a/lib/sqlalchemy/testing/assertions.py +++ b/lib/sqlalchemy/testing/assertions.py @@ -405,13 +405,16 @@ class AssertsExecutionResults(object): cls.__name__, repr(expected_item))) return True + def sql_execution_asserter(self, db=None): + if db is None: + from . import db as db + + return assertsql.assert_engine(db) + def assert_sql_execution(self, db, callable_, *rules): - assertsql.asserter.add_rules(rules) - try: + with self.sql_execution_asserter(db) as asserter: callable_() - assertsql.asserter.statement_complete() - finally: - assertsql.asserter.clear_rules() + asserter.assert_(*rules) def assert_sql(self, db, callable_, list_, with_sequences=None): if (with_sequences is not None and diff --git a/lib/sqlalchemy/testing/assertsql.py b/lib/sqlalchemy/testing/assertsql.py index bcc999fe3..2ac0605a2 100644 --- a/lib/sqlalchemy/testing/assertsql.py +++ b/lib/sqlalchemy/testing/assertsql.py @@ -8,6 +8,9 @@ from ..engine.default import DefaultDialect from .. import util import re +import collections +import contextlib +from .. import event class AssertRule(object): @@ -321,39 +324,78 @@ def _process_assertion_statement(query, context): return query -class SQLAssert(object): +class SQLExecuteObserved( + collections.namedtuple( + "SQLExecuteObserved", ["clauseelement", "multiparams", "params"]) +): + def process(self, rules): + if rules is not None: + if not rules: + assert False, \ + 'All rules have been exhausted, but further '\ + 'statements remain' + rule = rules[0] + rule.process_execute( + self.clauseelement, *self.multiparams, **self.params) + if rule.is_consumed(): + rules.pop(0) - rules = None - def add_rules(self, rules): - self.rules = list(rules) +class SQLCursorExecuteObserved( + collections.namedtuple( + "SQLCursorExecuteObserved", + ["statement", "parameters", "context", "executemany"]) +): + def process(self, rules): + if rules: + rule = rules[0] + rule.process_cursor_execute( + self.statement, self.parameters, + self.context, self.executemany) - def statement_complete(self): - for rule in self.rules: + +class SQLAsserter(object): + def __init__(self): + self.accumulated = [] + + def _close(self): + # safety feature in case event.remove + # goes haywire + self._final = self.accumulated + del self.accumulated + + def assert_(self, *rules): + rules = list(rules) + for observed in self._final: + observed.process(rules) + + for rule in rules: if not rule.consume_final(): assert False, \ 'All statements are complete, but pending '\ 'assertion rules remain' - def clear_rules(self): - del self.rules - def execute(self, conn, clauseelement, multiparams, params, result): - if self.rules is not None: - if not self.rules: - assert False, \ - 'All rules have been exhausted, but further '\ - 'statements remain' - rule = self.rules[0] - rule.process_execute(clauseelement, *multiparams, **params) - if rule.is_consumed(): - self.rules.pop(0) +@contextlib.contextmanager +def assert_engine(engine): + asserter = SQLAsserter() - def cursor_execute(self, conn, cursor, statement, parameters, - context, executemany): - if self.rules: - rule = self.rules[0] - rule.process_cursor_execute(statement, parameters, context, - executemany) + @event.listens_for(engine, "after_execute") + def execute(conn, clauseelement, multiparams, params, result): + asserter.accumulated.append( + SQLExecuteObserved( + clauseelement, multiparams, params)) -asserter = SQLAssert() + @event.listens_for(engine, "after_cursor_execute") + def cursor_execute(conn, cursor, statement, parameters, + context, executemany): + asserter.accumulated.append( + SQLCursorExecuteObserved( + statement, parameters, context, executemany)) + + try: + yield asserter + finally: + asserter._close() + event.remove(engine, "after_cursor_execute", cursor_execute) + event.remove(engine, "after_execute", execute) diff --git a/lib/sqlalchemy/testing/engines.py b/lib/sqlalchemy/testing/engines.py index 0f6f59401..7d73e7423 100644 --- a/lib/sqlalchemy/testing/engines.py +++ b/lib/sqlalchemy/testing/engines.py @@ -204,7 +204,6 @@ def testing_engine(url=None, options=None): """Produce an engine configured by --options with optional overrides.""" from sqlalchemy import create_engine - from .assertsql import asserter if not options: use_reaper = True @@ -219,8 +218,6 @@ def testing_engine(url=None, options=None): if isinstance(engine.pool, pool.QueuePool): engine.pool._timeout = 0 engine.pool._max_overflow = 0 - event.listen(engine, 'after_execute', asserter.execute) - event.listen(engine, 'after_cursor_execute', asserter.cursor_execute) if use_reaper: event.listen(engine.pool, 'connect', testing_reaper.connect) event.listen(engine.pool, 'checkout', testing_reaper.checkout) -- cgit v1.2.1 From e257ca6c5268517ec2e9a561372d82dfc10475e8 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 7 Dec 2014 18:55:23 -0500 Subject: - initial tests for bulk --- lib/sqlalchemy/orm/session.py | 3 +- test/orm/test_bulk.py | 317 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 test/orm/test_bulk.py diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index ef911824c..7dd577230 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -2056,7 +2056,8 @@ class Session(_SessionClassMethods): mapper, states, isupdate, True, return_defaults) def bulk_insert_mappings(self, mapper, mappings, return_defaults=False): - self._bulk_save_mappings(mapper, mappings, False, False, return_defaults) + self._bulk_save_mappings( + mapper, mappings, False, False, return_defaults) def bulk_update_mappings(self, mapper, mappings): self._bulk_save_mappings(mapper, mappings, True, False, False) diff --git a/test/orm/test_bulk.py b/test/orm/test_bulk.py new file mode 100644 index 000000000..4bcde2480 --- /dev/null +++ b/test/orm/test_bulk.py @@ -0,0 +1,317 @@ +from sqlalchemy import testing +from sqlalchemy.testing import eq_ +from sqlalchemy.testing.schema import Table, Column +from sqlalchemy.testing import fixtures +from sqlalchemy import Integer, String, ForeignKey +from sqlalchemy.orm import mapper, Session +from sqlalchemy.testing.assertsql import CompiledSQL +from test.orm import _fixtures + + +class BulkTest(testing.AssertsExecutionResults): + run_inserts = None + + +class BulkInsertTest(BulkTest, _fixtures.FixtureTest): + + @classmethod + def setup_mappers(cls): + User, Address = cls.classes("User", "Address") + u, a = cls.tables("users", "addresses") + + mapper(User, u) + mapper(Address, a) + + def test_bulk_save_return_defaults(self): + User, = self.classes("User",) + + s = Session() + objects = [ + User(name="u1"), + User(name="u2"), + User(name="u3") + ] + assert 'id' not in objects[0].__dict__ + + with self.sql_execution_asserter() as asserter: + s.bulk_save_objects(objects, return_defaults=True) + + asserter.assert_( + CompiledSQL( + "INSERT INTO users (name) VALUES (:name)", + [{'name': 'u1'}] + ), + CompiledSQL( + "INSERT INTO users (name) VALUES (:name)", + [{'name': 'u2'}] + ), + CompiledSQL( + "INSERT INTO users (name) VALUES (:name)", + [{'name': 'u3'}] + ), + ) + eq_(objects[0].__dict__['id'], 1) + + def test_bulk_save_no_defaults(self): + User, = self.classes("User",) + + s = Session() + objects = [ + User(name="u1"), + User(name="u2"), + User(name="u3") + ] + assert 'id' not in objects[0].__dict__ + + with self.sql_execution_asserter() as asserter: + s.bulk_save_objects(objects) + + asserter.assert_( + CompiledSQL( + "INSERT INTO users (name) VALUES (:name)", + [{'name': 'u1'}, {'name': 'u2'}, {'name': 'u3'}] + ), + ) + assert 'id' not in objects[0].__dict__ + + +class BulkInheritanceTest(fixtures.MappedTest, BulkTest): + @classmethod + def define_tables(cls, metadata): + Table( + 'people', metadata, + Column( + 'person_id', Integer, + primary_key=True, + test_needs_autoincrement=True), + Column('name', String(50)), + Column('type', String(30))) + + Table( + 'engineers', metadata, + Column( + 'person_id', Integer, + ForeignKey('people.person_id'), + primary_key=True), + Column('status', String(30)), + Column('primary_language', String(50))) + + Table( + 'managers', metadata, + Column( + 'person_id', Integer, + ForeignKey('people.person_id'), + primary_key=True), + Column('status', String(30)), + Column('manager_name', String(50))) + + Table( + 'boss', metadata, + Column( + 'boss_id', Integer, + ForeignKey('managers.person_id'), + primary_key=True), + Column('golf_swing', String(30))) + + @classmethod + def setup_classes(cls): + class Base(cls.Comparable): + pass + + class Person(Base): + pass + + class Engineer(Person): + pass + + class Manager(Person): + pass + + class Boss(Manager): + pass + + @classmethod + def setup_mappers(cls): + Person, Engineer, Manager, Boss = \ + cls.classes('Person', 'Engineer', 'Manager', 'Boss') + p, e, m, b = cls.tables('people', 'engineers', 'managers', 'boss') + + mapper( + Person, p, polymorphic_on=p.c.type, + polymorphic_identity='person') + mapper(Engineer, e, inherits=Person, polymorphic_identity='engineer') + mapper(Manager, m, inherits=Person, polymorphic_identity='manager') + mapper(Boss, b, inherits=Manager, polymorphic_identity='boss') + + def test_bulk_save_joined_inh_return_defaults(self): + Person, Engineer, Manager, Boss = \ + self.classes('Person', 'Engineer', 'Manager', 'Boss') + + s = Session() + objects = [ + Manager(name='m1', status='s1', manager_name='mn1'), + Engineer(name='e1', status='s2', primary_language='l1'), + Engineer(name='e2', status='s3', primary_language='l2'), + Boss( + name='b1', status='s3', manager_name='mn2', + golf_swing='g1') + ] + assert 'person_id' not in objects[0].__dict__ + + with self.sql_execution_asserter() as asserter: + s.bulk_save_objects(objects, return_defaults=True) + + asserter.assert_( + CompiledSQL( + "INSERT INTO people (name, type) VALUES (:name, :type)", + [{'type': 'manager', 'name': 'm1'}] + ), + CompiledSQL( + "INSERT INTO managers (person_id, status, manager_name) " + "VALUES (:person_id, :status, :manager_name)", + [{'person_id': 1, 'status': 's1', 'manager_name': 'mn1'}] + ), + CompiledSQL( + "INSERT INTO people (name, type) VALUES (:name, :type)", + [{'type': 'engineer', 'name': 'e1'}] + ), + CompiledSQL( + "INSERT INTO people (name, type) VALUES (:name, :type)", + [{'type': 'engineer', 'name': 'e2'}] + ), + CompiledSQL( + "INSERT INTO engineers (person_id, status, primary_language) " + "VALUES (:person_id, :status, :primary_language)", + [{'person_id': 2, 'status': 's2', 'primary_language': 'l1'}, + {'person_id': 3, 'status': 's3', 'primary_language': 'l2'}] + + ), + CompiledSQL( + "INSERT INTO people (name, type) VALUES (:name, :type)", + [{'type': 'boss', 'name': 'b1'}] + ), + CompiledSQL( + "INSERT INTO managers (person_id, status, manager_name) " + "VALUES (:person_id, :status, :manager_name)", + [{'person_id': 4, 'status': 's3', 'manager_name': 'mn2'}] + + ), + CompiledSQL( + "INSERT INTO boss (golf_swing) VALUES (:golf_swing)", + [{'golf_swing': 'g1'}] + ) + ) + eq_(objects[0].__dict__['person_id'], 1) + + def test_bulk_save_joined_inh_no_defaults(self): + Person, Engineer, Manager, Boss = \ + self.classes('Person', 'Engineer', 'Manager', 'Boss') + + s = Session() + with self.sql_execution_asserter() as asserter: + s.bulk_save_objects([ + Manager( + person_id=1, + name='m1', status='s1', manager_name='mn1'), + Engineer( + person_id=2, + name='e1', status='s2', primary_language='l1'), + Engineer( + person_id=3, + name='e2', status='s3', primary_language='l2'), + Boss( + person_id=4, + name='b1', status='s3', manager_name='mn2', + golf_swing='g1') + ], + + ) + + # the only difference here is that common classes are grouped together. + # at the moment it doesn't lump all the "people" tables from + # different classes together. + asserter.assert_( + CompiledSQL( + "INSERT INTO people (person_id, name, type) VALUES " + "(:person_id, :name, :type)", + [{'person_id': 1, 'type': 'manager', 'name': 'm1'}] + ), + CompiledSQL( + "INSERT INTO managers (person_id, status, manager_name) " + "VALUES (:person_id, :status, :manager_name)", + [{'status': 's1', 'person_id': 1, 'manager_name': 'mn1'}] + ), + CompiledSQL( + "INSERT INTO people (person_id, name, type) VALUES " + "(:person_id, :name, :type)", + [{'person_id': 2, 'type': 'engineer', 'name': 'e1'}, + {'person_id': 3, 'type': 'engineer', 'name': 'e2'}] + ), + CompiledSQL( + "INSERT INTO engineers (person_id, status, primary_language) " + "VALUES (:person_id, :status, :primary_language)", + [{'person_id': 2, 'status': 's2', 'primary_language': 'l1'}, + {'person_id': 3, 'status': 's3', 'primary_language': 'l2'}] + ), + CompiledSQL( + "INSERT INTO people (person_id, name, type) VALUES " + "(:person_id, :name, :type)", + [{'person_id': 4, 'type': 'boss', 'name': 'b1'}] + ), + CompiledSQL( + "INSERT INTO managers (person_id, status, manager_name) " + "VALUES (:person_id, :status, :manager_name)", + [{'status': 's3', 'person_id': 4, 'manager_name': 'mn2'}] + ), + CompiledSQL( + "INSERT INTO boss (golf_swing) VALUES (:golf_swing)", + [{'golf_swing': 'g1'}] + ) + ) + + def test_bulk_insert_joined_inh_return_defaults(self): + Person, Engineer, Manager, Boss = \ + self.classes('Person', 'Engineer', 'Manager', 'Boss') + + s = Session() + with self.sql_execution_asserter() as asserter: + s.bulk_insert_mappings( + Boss, + [ + dict( + name='b1', status='s1', manager_name='mn1', + golf_swing='g1' + ), + dict( + name='b2', status='s2', manager_name='mn2', + golf_swing='g2' + ), + dict( + name='b3', status='s3', manager_name='mn3', + golf_swing='g3' + ), + ] + ) + + # the only difference here is that common classes are grouped together. + # at the moment it doesn't lump all the "people" tables from + # different classes together. + asserter.assert_( + CompiledSQL( + "INSERT INTO people (name) VALUES (:name)", + [{'name': 'b1'}, {'name': 'b2'}, {'name': 'b3'}] + ), + CompiledSQL( + "INSERT INTO managers (status, manager_name) VALUES " + "(:status, :manager_name)", + [{'status': 's1', 'manager_name': 'mn1'}, + {'status': 's2', 'manager_name': 'mn2'}, + {'status': 's3', 'manager_name': 'mn3'}] + + ), + CompiledSQL( + "INSERT INTO boss (golf_swing) VALUES (:golf_swing)", + [{'golf_swing': 'g1'}, + {'golf_swing': 'g2'}, {'golf_swing': 'g3'}] + ) + ) -- cgit v1.2.1 From c42b8f8eb8f4c324e2469bf3baaa316c214abce5 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 7 Dec 2014 20:21:20 -0500 Subject: - fix inheritance persistence - start writing docs --- lib/sqlalchemy/orm/persistence.py | 15 ++-- lib/sqlalchemy/orm/session.py | 158 ++++++++++++++++++++++++++++++++++++++ lib/sqlalchemy/orm/sync.py | 17 ++++ test/orm/test_bulk.py | 50 +++++++----- 4 files changed, 215 insertions(+), 25 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 81024c41f..d94fbb040 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -49,7 +49,7 @@ def _bulk_insert( continue records = ( - (None, state_dict, params, super_mapper, + (None, state_dict, params, mapper, connection, value_params, has_all_pks, has_all_defaults) for state, state_dict, params, mp, @@ -918,7 +918,7 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, states): def _postfetch(mapper, uowtransaction, table, - state, dict_, result, params, value_params): + state, dict_, result, params, value_params, bulk=False): """Expire attributes in need of newly persisted database state, after an INSERT or UPDATE statement has proceeded for that state.""" @@ -954,10 +954,13 @@ def _postfetch(mapper, uowtransaction, table, # TODO: this still goes a little too often. would be nice to # have definitive list of "columns that changed" here for m, equated_pairs in mapper._table_to_equated[table]: - sync.populate(state, m, state, m, - equated_pairs, - uowtransaction, - mapper.passive_updates) + if state is None: + sync.bulk_populate_inherit_keys(dict_, m, equated_pairs) + else: + sync.populate(state, m, state, m, + equated_pairs, + uowtransaction, + mapper.passive_updates) def _connections_for_states(base_mapper, uowtransaction, states): diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 7dd577230..e07b4554e 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -2048,6 +2048,66 @@ class Session(_SessionClassMethods): transaction.rollback(_capture_exception=True) def bulk_save_objects(self, objects, return_defaults=False): + """Perform a bulk save of the given list of objects. + + The bulk save feature allows mapped objects to be used as the + source of simple INSERT and UPDATE operations which can be more easily + grouped together into higher performing "executemany" + operations; the extraction of data from the objects is also performed + using a lower-latency process that ignores whether or not attributes + have actually been modified in the case of UPDATEs, and also ignores + SQL expressions. + + The objects as given are not added to the session and no additional + state is established on them, unless the ``return_defaults`` flag + is also set. + + .. warning:: + + The bulk save feature allows for a lower-latency INSERT/UPDATE + of rows at the expense of a lack of features. Features such + as object management, relationship handling, and SQL clause + support are bypassed in favor of raw INSERT/UPDATES of records. + + **Please read the list of caveats at :ref:`bulk_operations` + before using this method.** + + :param objects: a list of mapped object instances. The mapped + objects are persisted as is, and are **not** associated with the + :class:`.Session` afterwards. + + For each object, whether the object is sent as an INSERT or an + UPDATE is dependent on the same rules used by the :class:`.Session` + in traditional operation; if the object has the + :attr:`.InstanceState.key` + attribute set, then the object is assumed to be "detached" and + will result in an UPDATE. Otherwise, an INSERT is used. + + In the case of an UPDATE, **all** those attributes which are present + and are not part of the primary key are applied to the SET clause + of the UPDATE statement, regardless of whether any change in state + was logged on each attribute; there is no checking of per-attribute + history. The primary key attributes, which are required, + are applied to the WHERE clause. + + :param return_defaults: when True, rows that are missing values which + generate defaults, namely integer primary key defaults and sequences, + will be inserted **one at a time**, so that the primary key value + is available. In particular this will allow joined-inheritance + and other multi-table mappings to insert correctly without the need + to provide primary key values ahead of time; however, + return_defaults mode greatly reduces the performance gains of the + method overall. + + .. seealso:: + + :ref:`bulk_operations` + + :meth:`.Session.bulk_insert_mappings` + + :meth:`.Session.bulk_update_mappings` + + """ for (mapper, isupdate), states in itertools.groupby( (attributes.instance_state(obj) for obj in objects), lambda state: (state.mapper, state.key is not None) @@ -2056,10 +2116,108 @@ class Session(_SessionClassMethods): mapper, states, isupdate, True, return_defaults) def bulk_insert_mappings(self, mapper, mappings, return_defaults=False): + """Perform a bulk insert of the given list of mapping dictionaries. + + The bulk insert feature allows plain Python dictionaries to be used as + the source of simple INSERT operations which can be more easily + grouped together into higher performing "executemany" + operations. Using dictionaries, there is no "history" or session + state management features in use, reducing latency when inserting + large numbers of simple rows. + + The values within the dictionaries as given are typically passed + without modification into Core :meth:`.Insert` constructs, after + organizing the values within them across the tables to which + the given mapper is mapped. + + .. warning:: + + The bulk insert feature allows for a lower-latency INSERT + of rows at the expense of a lack of features. Features such + as relationship handling and SQL clause support are bypassed + in favor of a raw INSERT of records. + + **Please read the list of caveats at :ref:`bulk_operations` + before using this method.** + + :param mapper: a mapped class, or the actual :class:`.Mapper` object, + representing the single kind of object represented within the mapping + list. + + :param mappings: a list of dictionaries, each one containing the state + of the mapped row to be inserted, in terms of the attribute names + on the mapped class. If the mapping refers to multiple tables, + such as a joined-inheritance mapping, each dictionary must contain + all keys to be populated into all tables. + + :param return_defaults: when True, rows that are missing values which + generate defaults, namely integer primary key defaults and sequences, + will be inserted **one at a time**, so that the primary key value + is available. In particular this will allow joined-inheritance + and other multi-table mappings to insert correctly without the need + to provide primary + key values ahead of time; however, return_defaults mode greatly + reduces the performance gains of the method overall. If the rows + to be inserted only refer to a single table, then there is no + reason this flag should be set as the returned default information + is not used. + + + .. seealso:: + + :ref:`bulk_operations` + + :meth:`.Session.bulk_save_objects` + + :meth:`.Session.bulk_update_mappings` + + """ self._bulk_save_mappings( mapper, mappings, False, False, return_defaults) def bulk_update_mappings(self, mapper, mappings): + """Perform a bulk update of the given list of mapping dictionaries. + + The bulk update feature allows plain Python dictionaries to be used as + the source of simple UPDATE operations which can be more easily + grouped together into higher performing "executemany" + operations. Using dictionaries, there is no "history" or session + state management features in use, reducing latency when updating + large numbers of simple rows. + + .. warning:: + + The bulk update feature allows for a lower-latency UPDATE + of rows at the expense of a lack of features. Features such + as relationship handling and SQL clause support are bypassed + in favor of a raw UPDATE of records. + + **Please read the list of caveats at :ref:`bulk_operations` + before using this method.** + + :param mapper: a mapped class, or the actual :class:`.Mapper` object, + representing the single kind of object represented within the mapping + list. + + :param mappings: a list of dictionaries, each one containing the state + of the mapped row to be updated, in terms of the attribute names + on the mapped class. If the mapping refers to multiple tables, + such as a joined-inheritance mapping, each dictionary may contain + keys corresponding to all tables. All those keys which are present + and are not part of the primary key are applied to the SET clause + of the UPDATE statement; the primary key values, which are required, + are applied to the WHERE clause. + + + .. seealso:: + + :ref:`bulk_operations` + + :meth:`.Session.bulk_insert_mappings` + + :meth:`.Session.bulk_save_objects` + + """ self._bulk_save_mappings(mapper, mappings, True, False, False) def _bulk_save_mappings( diff --git a/lib/sqlalchemy/orm/sync.py b/lib/sqlalchemy/orm/sync.py index e1ef85c1d..671c7c067 100644 --- a/lib/sqlalchemy/orm/sync.py +++ b/lib/sqlalchemy/orm/sync.py @@ -45,6 +45,23 @@ def populate(source, source_mapper, dest, dest_mapper, uowcommit.attributes[("pk_cascaded", dest, r)] = True +def bulk_populate_inherit_keys( + source_dict, source_mapper, synchronize_pairs): + # a simplified version of populate() used by bulk insert mode + for l, r in synchronize_pairs: + try: + prop = source_mapper._columntoproperty[l] + value = source_dict[prop.key] + except exc.UnmappedColumnError: + _raise_col_to_prop(False, source_mapper, l, source_mapper, r) + + try: + prop = source_mapper._columntoproperty[r] + source_dict[prop.key] = value + except exc.UnmappedColumnError: + _raise_col_to_prop(True, source_mapper, l, source_mapper, r) + + def clear(dest, dest_mapper, synchronize_pairs): for l, r in synchronize_pairs: if r.primary_key and \ diff --git a/test/orm/test_bulk.py b/test/orm/test_bulk.py index 4bcde2480..f6d2513d1 100644 --- a/test/orm/test_bulk.py +++ b/test/orm/test_bulk.py @@ -10,6 +10,7 @@ from test.orm import _fixtures class BulkTest(testing.AssertsExecutionResults): run_inserts = None + run_define_tables = 'each' class BulkInsertTest(BulkTest, _fixtures.FixtureTest): @@ -75,7 +76,7 @@ class BulkInsertTest(BulkTest, _fixtures.FixtureTest): assert 'id' not in objects[0].__dict__ -class BulkInheritanceTest(fixtures.MappedTest, BulkTest): +class BulkInheritanceTest(BulkTest, fixtures.MappedTest): @classmethod def define_tables(cls, metadata): Table( @@ -197,11 +198,14 @@ class BulkInheritanceTest(fixtures.MappedTest, BulkTest): ), CompiledSQL( - "INSERT INTO boss (golf_swing) VALUES (:golf_swing)", - [{'golf_swing': 'g1'}] + "INSERT INTO boss (boss_id, golf_swing) VALUES " + "(:boss_id, :golf_swing)", + [{'boss_id': 4, 'golf_swing': 'g1'}] ) ) eq_(objects[0].__dict__['person_id'], 1) + eq_(objects[3].__dict__['person_id'], 4) + eq_(objects[3].__dict__['boss_id'], 4) def test_bulk_save_joined_inh_no_defaults(self): Person, Engineer, Manager, Boss = \ @@ -220,7 +224,7 @@ class BulkInheritanceTest(fixtures.MappedTest, BulkTest): person_id=3, name='e2', status='s3', primary_language='l2'), Boss( - person_id=4, + person_id=4, boss_id=4, name='b1', status='s3', manager_name='mn2', golf_swing='g1') ], @@ -264,8 +268,9 @@ class BulkInheritanceTest(fixtures.MappedTest, BulkTest): [{'status': 's3', 'person_id': 4, 'manager_name': 'mn2'}] ), CompiledSQL( - "INSERT INTO boss (golf_swing) VALUES (:golf_swing)", - [{'golf_swing': 'g1'}] + "INSERT INTO boss (boss_id, golf_swing) VALUES " + "(:boss_id, :golf_swing)", + [{'boss_id': 4, 'golf_swing': 'g1'}] ) ) @@ -290,28 +295,35 @@ class BulkInheritanceTest(fixtures.MappedTest, BulkTest): name='b3', status='s3', manager_name='mn3', golf_swing='g3' ), - ] + ], return_defaults=True ) - # the only difference here is that common classes are grouped together. - # at the moment it doesn't lump all the "people" tables from - # different classes together. asserter.assert_( CompiledSQL( "INSERT INTO people (name) VALUES (:name)", - [{'name': 'b1'}, {'name': 'b2'}, {'name': 'b3'}] + [{'name': 'b1'}] + ), + CompiledSQL( + "INSERT INTO people (name) VALUES (:name)", + [{'name': 'b2'}] ), CompiledSQL( - "INSERT INTO managers (status, manager_name) VALUES " - "(:status, :manager_name)", - [{'status': 's1', 'manager_name': 'mn1'}, - {'status': 's2', 'manager_name': 'mn2'}, - {'status': 's3', 'manager_name': 'mn3'}] + "INSERT INTO people (name) VALUES (:name)", + [{'name': 'b3'}] + ), + CompiledSQL( + "INSERT INTO managers (person_id, status, manager_name) " + "VALUES (:person_id, :status, :manager_name)", + [{'person_id': 1, 'status': 's1', 'manager_name': 'mn1'}, + {'person_id': 2, 'status': 's2', 'manager_name': 'mn2'}, + {'person_id': 3, 'status': 's3', 'manager_name': 'mn3'}] ), CompiledSQL( - "INSERT INTO boss (golf_swing) VALUES (:golf_swing)", - [{'golf_swing': 'g1'}, - {'golf_swing': 'g2'}, {'golf_swing': 'g3'}] + "INSERT INTO boss (boss_id, golf_swing) VALUES " + "(:boss_id, :golf_swing)", + [{'golf_swing': 'g1', 'boss_id': 1}, + {'golf_swing': 'g2', 'boss_id': 2}, + {'golf_swing': 'g3', 'boss_id': 3}] ) ) -- cgit v1.2.1 From 07cc9e054ae4d5bb9cfc3c1d807b2a0d58a95b69 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 7 Dec 2014 20:36:01 -0500 Subject: - add an option for bulk_save -> update to not do history --- lib/sqlalchemy/orm/persistence.py | 9 +++++++-- lib/sqlalchemy/orm/session.py | 32 +++++++++++++++++++++----------- test/orm/test_bulk.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index d94fbb040..f477e1dd7 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -75,7 +75,8 @@ def _bulk_insert( ) -def _bulk_update(mapper, mappings, session_transaction, isstates): +def _bulk_update(mapper, mappings, session_transaction, + isstates, update_changed_only): base_mapper = mapper.base_mapper cached_connections = _cached_connection_dict(base_mapper) @@ -88,7 +89,10 @@ def _bulk_update(mapper, mappings, session_transaction, isstates): ) if isstates: - mappings = [_changed_dict(mapper, state) for state in mappings] + if update_changed_only: + mappings = [_changed_dict(mapper, state) for state in mappings] + else: + mappings = [state.dict for state in mappings] else: mappings = list(mappings) @@ -612,6 +616,7 @@ def _emit_update_statements(base_mapper, uowtransaction, rows = 0 records = list(records) + if hasvalue: for state, state_dict, params, mapper, \ connection, value_params in records: diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index e07b4554e..72d393f54 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -2047,7 +2047,8 @@ class Session(_SessionClassMethods): with util.safe_reraise(): transaction.rollback(_capture_exception=True) - def bulk_save_objects(self, objects, return_defaults=False): + def bulk_save_objects( + self, objects, return_defaults=False, update_changed_only=True): """Perform a bulk save of the given list of objects. The bulk save feature allows mapped objects to be used as the @@ -2083,12 +2084,13 @@ class Session(_SessionClassMethods): attribute set, then the object is assumed to be "detached" and will result in an UPDATE. Otherwise, an INSERT is used. - In the case of an UPDATE, **all** those attributes which are present - and are not part of the primary key are applied to the SET clause - of the UPDATE statement, regardless of whether any change in state - was logged on each attribute; there is no checking of per-attribute - history. The primary key attributes, which are required, - are applied to the WHERE clause. + In the case of an UPDATE, statements are grouped based on which + attributes have changed, and are thus to be the subject of each + SET clause. If ``update_changed_only`` is False, then all + attributes present within each object are applied to the UPDATE + statement, which may help in allowing the statements to be grouped + together into a larger executemany(), and will also reduce the + overhead of checking history on attributes. :param return_defaults: when True, rows that are missing values which generate defaults, namely integer primary key defaults and sequences, @@ -2099,6 +2101,11 @@ class Session(_SessionClassMethods): return_defaults mode greatly reduces the performance gains of the method overall. + :param update_changed_only: when True, UPDATE statements are rendered + based on those attributes in each state that have logged changes. + When False, all attributes present are rendered into the SET clause + with the exception of primary key attributes. + .. seealso:: :ref:`bulk_operations` @@ -2113,7 +2120,8 @@ class Session(_SessionClassMethods): lambda state: (state.mapper, state.key is not None) ): self._bulk_save_mappings( - mapper, states, isupdate, True, return_defaults) + mapper, states, isupdate, True, + return_defaults, update_changed_only) def bulk_insert_mappings(self, mapper, mappings, return_defaults=False): """Perform a bulk insert of the given list of mapping dictionaries. @@ -2218,10 +2226,11 @@ class Session(_SessionClassMethods): :meth:`.Session.bulk_save_objects` """ - self._bulk_save_mappings(mapper, mappings, True, False, False) + self._bulk_save_mappings(mapper, mappings, True, False, False, False) def _bulk_save_mappings( - self, mapper, mappings, isupdate, isstates, return_defaults): + self, mapper, mappings, isupdate, isstates, + return_defaults, update_changed_only): mapper = _class_to_mapper(mapper) self._flushing = True @@ -2230,7 +2239,8 @@ class Session(_SessionClassMethods): try: if isupdate: persistence._bulk_update( - mapper, mappings, transaction, isstates) + mapper, mappings, transaction, + isstates, update_changed_only) else: persistence._bulk_insert( mapper, mappings, transaction, isstates, return_defaults) diff --git a/test/orm/test_bulk.py b/test/orm/test_bulk.py index f6d2513d1..e27d3b73c 100644 --- a/test/orm/test_bulk.py +++ b/test/orm/test_bulk.py @@ -13,7 +13,7 @@ class BulkTest(testing.AssertsExecutionResults): run_define_tables = 'each' -class BulkInsertTest(BulkTest, _fixtures.FixtureTest): +class BulkInsertUpdateTest(BulkTest, _fixtures.FixtureTest): @classmethod def setup_mappers(cls): @@ -75,6 +75,35 @@ class BulkInsertTest(BulkTest, _fixtures.FixtureTest): ) assert 'id' not in objects[0].__dict__ + def test_bulk_save_updated_include_unchanged(self): + User, = self.classes("User",) + + s = Session(expire_on_commit=False) + objects = [ + User(name="u1"), + User(name="u2"), + User(name="u3") + ] + s.add_all(objects) + s.commit() + + objects[0].name = 'u1new' + objects[2].name = 'u3new' + + s = Session() + with self.sql_execution_asserter() as asserter: + s.bulk_save_objects(objects, update_changed_only=False) + + asserter.assert_( + CompiledSQL( + "UPDATE users SET id=:id, name=:name WHERE " + "users.id = :users_id", + [{'users_id': 1, 'id': 1, 'name': 'u1new'}, + {'users_id': 2, 'id': 2, 'name': 'u2'}, + {'users_id': 3, 'id': 3, 'name': 'u3new'}] + ) + ) + class BulkInheritanceTest(BulkTest, fixtures.MappedTest): @classmethod -- cgit v1.2.1 From 5ed7a9672a4c143f111a15f26dfce4bc80547b6f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 7 Dec 2014 21:08:14 -0500 Subject: start docs... --- doc/build/orm/session.rst | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/doc/build/orm/session.rst b/doc/build/orm/session.rst index 78ae1ba81..01ac7230e 100644 --- a/doc/build/orm/session.rst +++ b/doc/build/orm/session.rst @@ -2456,6 +2456,61 @@ tables) across multiple databases. See the "sharding" example: :ref:`examples_sharding`. +.. _bulk_operations: + +Bulk Operations +--------------- + +.. note:: Bulk Operations mode is a new series of operations made available + on the :class:`.Session` object for the purpose of invoking INSERT and + UPDATE statements with greatly reduced Python overhead, at the expense + of much less functionality, automation, and error checking. + As of SQLAlchemy 1.0, these features should be considered as "beta", and + additionally are intended for advanced users. + +.. versionadded:: 1.0.0 + +Bulk operations on the :class:`.Session` include :meth:`.Session.bulk_save_objects`, +:meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings`. +The purpose of these methods is to directly expose internal elements of the unit of work system, +such that facilities for emitting INSERT and UPDATE statements given dictionaries +or object states can be utilized alone, bypassing the normal unit of work +mechanics of state, relationship and attribute management. The advantages +to this approach is strictly one of reduced Python overhead: + +* The flush() process, including the survey of all objects, their state, + their cascade status, the status of all objects associated with them + via :meth:`.relationship`, and the topological sort of all operations to + be performed is completely bypassed. This reduces a great amount of + Python overhead. + +* The objects as given have no defined relationship to the target + :class:`.Session`, even when the operation is complete, meaning there's no + overhead in attaching them or managing their state in terms of the identity + map or session. + +* The :meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings` + methods accept lists of plain Python dictionaries, not objects; this further + reduces a large amount of overhead associated with instantiating mapped + objects and assigning state to them, which normally is also subject to + expensive tracking of history on a per-attribute basis. + +* The process of fetching primary keys after an INSERT also is disabled by + default. When performed correctly, INSERT statements can now more readily + be batched by the unit of work process into ``executemany()`` blocks, which + perform vastly better than individual statement invocations. + +* UPDATE statements can similarly be tailored such that all attributes + are subject to the SET clase unconditionally, again making it much more + likely that ``executemany()`` blocks can be used. + +The performance behavior of the bulk routines should be studied using the +:ref:`examples_performance` example suite. This is a series of example +scripts which illustrate Python call-counts across a variety of scenarios, +including bulk insert and update scenarios. + + + Sessions API ============ -- cgit v1.2.1 From 3f1477e2ecf3b2e95a26383490d0e8c363f4d0cc Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 8 Dec 2014 01:10:30 -0500 Subject: - A new series of :class:`.Session` methods which provide hooks directly into the unit of work's facility for emitting INSERT and UPDATE statements has been created. When used correctly, this expert-oriented system can allow ORM-mappings to be used to generate bulk insert and update statements batched into executemany groups, allowing the statements to proceed at speeds that rival direct use of the Core. fixes #3100 --- doc/build/changelog/changelog_10.rst | 29 ++++++++++++ doc/build/changelog/migration_10.rst | 35 +++++++++++++- doc/build/core/tutorial.rst | 2 + doc/build/faq.rst | 29 ++++++------ doc/build/orm/session.rst | 92 ++++++++++++++++++++++++++++++++++-- examples/performance/__init__.py | 9 ++++ examples/performance/__main__.py | 2 + lib/sqlalchemy/orm/session.py | 59 ++++++++++++++--------- 8 files changed, 217 insertions(+), 40 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 6d99095d9..d6f36e97e 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,35 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, examples + + A new suite of examples dedicated to providing a detailed study + into performance of SQLAlchemy ORM and Core, as well as the DBAPI, + from multiple perspectives. The suite runs within a container + that provides built in profiling displays both through console + output as well as graphically via the RunSnake tool. + + .. seealso:: + + :ref:`examples_performance` + + .. change:: + :tags: feature, orm + :tickets: 3100 + + A new series of :class:`.Session` methods which provide hooks + directly into the unit of work's facility for emitting INSERT + and UPDATE statements has been created. When used correctly, + this expert-oriented system can allow ORM-mappings to be used + to generate bulk insert and update statements batched into + executemany groups, allowing the statements to proceed at + speeds that rival direct use of the Core. + + .. seealso:: + + :ref:`bulk_operations` + .. change:: :tags: feature, mssql :tickets: 3039 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 562bb9f1b..cd5d420e5 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -8,7 +8,7 @@ What's New in SQLAlchemy 1.0? undergoing maintenance releases as of May, 2014, and SQLAlchemy version 1.0, as of yet unreleased. - Document last updated: October 23, 2014 + Document last updated: December 8, 2014 Introduction ============ @@ -230,6 +230,39 @@ the :class:`.Table` construct. :ticket:`2051` +New Session Bulk INSERT/UPDATE API +---------------------------------- + +A new series of :class:`.Session` methods which provide hooks directly +into the unit of work's facility for emitting INSERT and UPDATE +statements has been created. When used correctly, this expert-oriented system +can allow ORM-mappings to be used to generate bulk insert and update +statements batched into executemany groups, allowing the statements +to proceed at speeds that rival direct use of the Core. + +.. seealso:: + + :ref:`bulk_operations` - introduction and full documentation + +:ticket:`3100` + +New Performance Example Suite +------------------------------ + +Inspired by the benchmarking done for the :ref:`bulk_operations` feature +as well as for the :ref:`faq_how_to_profile` section of the FAQ, a new +example section has been added which features several scripts designed +to illustrate the relative performance profile of various Core and ORM +techniques. The scripts are organized into use cases, and are packaged +under a single console interface such that any combination of demonstrations +can be run, dumping out timings, Python profile results and/or RunSnake profile +displays. + +.. seealso:: + + :ref:`examples_performance` + + .. _feature_get_enums: New get_enums() method with Postgresql Dialect diff --git a/doc/build/core/tutorial.rst b/doc/build/core/tutorial.rst index 04a25b174..b6c07bdc0 100644 --- a/doc/build/core/tutorial.rst +++ b/doc/build/core/tutorial.rst @@ -307,6 +307,8 @@ them is different across different databases; each database's determine the correct value (or values; note that ``inserted_primary_key`` returns a list so that it supports composite primary keys). +.. _execute_multiple: + Executing Multiple Statements ============================== diff --git a/doc/build/faq.rst b/doc/build/faq.rst index 555fdc9e1..8c3bd24f4 100644 --- a/doc/build/faq.rst +++ b/doc/build/faq.rst @@ -705,9 +705,13 @@ main query. :ref:`subqueryload_ordering` +.. _faq_performance: + Performance =========== +.. _faq_how_to_profile: + How can I profile a SQLAlchemy powered application? --------------------------------------------------- @@ -961,18 +965,10 @@ Common strategies to mitigate this include: The output of a profile can be a little daunting but after some practice they are very easy to read. -If you're feeling ambitious, there's also a more involved example of -SQLAlchemy profiling within the SQLAlchemy unit tests in the -``tests/aaa_profiling`` section. Tests in this area -use decorators that assert a -maximum number of method calls being used for particular operations, -so that if something inefficient gets checked in, the tests will -reveal it (it is important to note that in cPython, function calls have -the highest overhead of any operation, and the count of calls is more -often than not nearly proportional to time spent). Of note are the -the "zoomark" tests which use a fancy "SQL capturing" scheme which -cuts out the overhead of the DBAPI from the equation - although that -technique isn't really necessary for garden-variety profiling. +.. seealso:: + + :ref:`examples_performance` - a suite of performance demonstrations + with bundled profiling capabilities. I'm inserting 400,000 rows with the ORM and it's really slow! -------------------------------------------------------------- @@ -1001,10 +997,15 @@ ORM as a first-class component. For the use case of fast bulk inserts, the SQL generation and execution system that the ORM builds on top of -is part of the Core. Using this system directly, we can produce an INSERT that +is part of the :doc:`Core `. Using this system directly, we can produce an INSERT that is competitive with using the raw database API directly. -The example below illustrates time-based tests for four different +Alternatively, the SQLAlchemy ORM offers the :ref:`bulk_operations` +suite of methods, which provide hooks into subsections of the unit of +work process in order to emit Core-level INSERT and UPDATE constructs with +a small degree of ORM-based automation. + +The example below illustrates time-based tests for several different methods of inserting rows, going from the most automated to the least. With cPython 2.7, runtimes observed:: diff --git a/doc/build/orm/session.rst b/doc/build/orm/session.rst index 01ac7230e..08ef9303e 100644 --- a/doc/build/orm/session.rst +++ b/doc/build/orm/session.rst @@ -1944,6 +1944,8 @@ transactions set the flag ``twophase=True`` on the session:: # before committing both transactions session.commit() +.. _session_sql_expressions: + Embedding SQL Insert/Update Expressions into a Flush ===================================================== @@ -2459,7 +2461,7 @@ See the "sharding" example: :ref:`examples_sharding`. .. _bulk_operations: Bulk Operations ---------------- +=============== .. note:: Bulk Operations mode is a new series of operations made available on the :class:`.Session` object for the purpose of invoking INSERT and @@ -2480,7 +2482,7 @@ to this approach is strictly one of reduced Python overhead: * The flush() process, including the survey of all objects, their state, their cascade status, the status of all objects associated with them - via :meth:`.relationship`, and the topological sort of all operations to + via :func:`.relationship`, and the topological sort of all operations to be performed is completely bypassed. This reduces a great amount of Python overhead. @@ -2489,7 +2491,7 @@ to this approach is strictly one of reduced Python overhead: overhead in attaching them or managing their state in terms of the identity map or session. -* The :meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings` +* The :meth:`.Session.bulk_insert_mappings` and :meth:`.Session.bulk_update_mappings` methods accept lists of plain Python dictionaries, not objects; this further reduces a large amount of overhead associated with instantiating mapped objects and assigning state to them, which normally is also subject to @@ -2509,6 +2511,90 @@ The performance behavior of the bulk routines should be studied using the scripts which illustrate Python call-counts across a variety of scenarios, including bulk insert and update scenarios. +.. seealso:: + + :ref:`examples_performance` - includes detailed examples of bulk operations + contrasted against traditional Core and ORM methods, including performance + metrics. + +Usage +----- + +The methods each work in the context of the :class:`.Session` object's +transaction, like any other:: + + s = Session() + objects = [ + User(name="u1"), + User(name="u2"), + User(name="u3") + ] + s.bulk_save_objects(objects) + +For :meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings`, +dictionaries are passed:: + + s.bulk_insert_mappings(User, + [dict(name="u1"), dict(name="u2"), dict(name="u3")] + ) + +.. seealso:: + + :meth:`.Session.bulk_save_objects` + + :meth:`.Session.bulk_insert_mappings` + + :meth:`.Session.bulk_update_mappings` + + +Comparison to Core Insert / Update Constructs +--------------------------------------------- + +The bulk methods offer performance that under particular circumstances +can be close to that of using the core :class:`.Insert` and +:class:`.Update` constructs in an "executemany" context (for a description +of "executemany", see :ref:`execute_multiple` in the Core tutorial). +In order to achieve this, the +:paramref:`.Session.bulk_insert_mappings.return_defaults` +flag should be disabled so that rows can be batched together. The example +suite in :ref:`examples_performance` should be carefully studied in order +to gain familiarity with how fast bulk performance can be achieved. + +ORM Compatibility +----------------- + +The bulk insert / update methods lose a significant amount of functionality +versus traditional ORM use. The following is a listing of features that +are **not available** when using these methods: + +* persistence along :meth:`.relationship` linkages + +* sorting of rows within order of dependency; rows are inserted or updated + directly in the order in which they are passed to the methods + +* Session-management on the given objects, including attachment to the + session, identity map management. + +* Functionality related to primary key mutation, ON UPDATE cascade + +* SQL expression inserts / updates (e.g. :ref:`session_sql_expressions`) + +* ORM events such as :meth:`.MapperEvents.before_insert`, etc. The bulk + session methods have no event support. + +Features that **are available** include:: + +* INSERTs and UPDATEs of mapped objects + +* Version identifier support + +* Multi-table mappings, such as joined-inheritance - however, an object + to be inserted across multiple tables either needs to have primary key + identifiers fully populated ahead of time, else the + :paramref:`.Session.bulk_save_objects.return_defaults` flag must be used, + which will greatly reduce the performance benefits + + Sessions API diff --git a/examples/performance/__init__.py b/examples/performance/__init__.py index 6e2e1fc89..a4edfce36 100644 --- a/examples/performance/__init__.py +++ b/examples/performance/__init__.py @@ -48,6 +48,15 @@ Or with options:: --dburl mysql+mysqldb://scott:tiger@localhost/test \\ --profile --num 1000 +.. seealso:: + + :ref:`faq_how_to_profile` + +File Listing +------------- + +.. autosource:: + Running all tests with time --------------------------- diff --git a/examples/performance/__main__.py b/examples/performance/__main__.py index 957d6c699..5e05143bf 100644 --- a/examples/performance/__main__.py +++ b/examples/performance/__main__.py @@ -1,3 +1,5 @@ +"""Allows the examples/performance package to be run as a script.""" + from . import Profiler if __name__ == '__main__': diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 72d393f54..d40d28154 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -2061,17 +2061,22 @@ class Session(_SessionClassMethods): The objects as given are not added to the session and no additional state is established on them, unless the ``return_defaults`` flag - is also set. + is also set, in which case primary key attributes and server-side + default values will be populated. + + .. versionadded:: 1.0.0 .. warning:: The bulk save feature allows for a lower-latency INSERT/UPDATE - of rows at the expense of a lack of features. Features such - as object management, relationship handling, and SQL clause - support are bypassed in favor of raw INSERT/UPDATES of records. + of rows at the expense of most other unit-of-work features. + Features such as object management, relationship handling, + and SQL clause support are **silently omitted** in favor of raw + INSERT/UPDATES of records. - **Please read the list of caveats at :ref:`bulk_operations` - before using this method.** + **Please read the list of caveats at** :ref:`bulk_operations` + **before using this method, and fully test and confirm the + functionality of all code developed using these systems.** :param objects: a list of mapped object instances. The mapped objects are persisted as is, and are **not** associated with the @@ -2098,8 +2103,8 @@ class Session(_SessionClassMethods): is available. In particular this will allow joined-inheritance and other multi-table mappings to insert correctly without the need to provide primary key values ahead of time; however, - return_defaults mode greatly reduces the performance gains of the - method overall. + :paramref:`.Session.bulk_save_objects.return_defaults` **greatly + reduces the performance gains** of the method overall. :param update_changed_only: when True, UPDATE statements are rendered based on those attributes in each state that have logged changes. @@ -2138,15 +2143,19 @@ class Session(_SessionClassMethods): organizing the values within them across the tables to which the given mapper is mapped. + .. versionadded:: 1.0.0 + .. warning:: The bulk insert feature allows for a lower-latency INSERT - of rows at the expense of a lack of features. Features such - as relationship handling and SQL clause support are bypassed - in favor of a raw INSERT of records. + of rows at the expense of most other unit-of-work features. + Features such as object management, relationship handling, + and SQL clause support are **silently omitted** in favor of raw + INSERT of records. - **Please read the list of caveats at :ref:`bulk_operations` - before using this method.** + **Please read the list of caveats at** :ref:`bulk_operations` + **before using this method, and fully test and confirm the + functionality of all code developed using these systems.** :param mapper: a mapped class, or the actual :class:`.Mapper` object, representing the single kind of object represented within the mapping @@ -2164,8 +2173,10 @@ class Session(_SessionClassMethods): is available. In particular this will allow joined-inheritance and other multi-table mappings to insert correctly without the need to provide primary - key values ahead of time; however, return_defaults mode greatly - reduces the performance gains of the method overall. If the rows + key values ahead of time; however, + :paramref:`.Session.bulk_insert_mappings.return_defaults` + **greatly reduces the performance gains** of the method overall. + If the rows to be inserted only refer to a single table, then there is no reason this flag should be set as the returned default information is not used. @@ -2181,7 +2192,7 @@ class Session(_SessionClassMethods): """ self._bulk_save_mappings( - mapper, mappings, False, False, return_defaults) + mapper, mappings, False, False, return_defaults, False) def bulk_update_mappings(self, mapper, mappings): """Perform a bulk update of the given list of mapping dictionaries. @@ -2193,15 +2204,19 @@ class Session(_SessionClassMethods): state management features in use, reducing latency when updating large numbers of simple rows. + .. versionadded:: 1.0.0 + .. warning:: The bulk update feature allows for a lower-latency UPDATE - of rows at the expense of a lack of features. Features such - as relationship handling and SQL clause support are bypassed - in favor of a raw UPDATE of records. - - **Please read the list of caveats at :ref:`bulk_operations` - before using this method.** + of rows at the expense of most other unit-of-work features. + Features such as object management, relationship handling, + and SQL clause support are **silently omitted** in favor of raw + UPDATES of records. + + **Please read the list of caveats at** :ref:`bulk_operations` + **before using this method, and fully test and confirm the + functionality of all code developed using these systems.** :param mapper: a mapped class, or the actual :class:`.Mapper` object, representing the single kind of object represented within the mapping -- cgit v1.2.1 From 902c8d480beebb69e09ee613fe51579c3fd2ce0d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 8 Dec 2014 01:18:07 -0500 Subject: - some profile changes likely due to the change in event listening on engines --- test/ext/test_horizontal_shard.py | 2 -- test/profiles.txt | 75 ++++++++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/test/ext/test_horizontal_shard.py b/test/ext/test_horizontal_shard.py index 99879a74d..0af33ecde 100644 --- a/test/ext/test_horizontal_shard.py +++ b/test/ext/test_horizontal_shard.py @@ -235,8 +235,6 @@ class AttachedFileShardTest(ShardTest, fixtures.TestBase): def _init_dbs(self): db1 = testing_engine('sqlite://', options={"execution_options": {"shard_id": "shard1"}}) - assert db1._has_events - db2 = db1.execution_options(shard_id="shard2") db3 = db1.execution_options(shard_id="shard3") db4 = db1.execution_options(shard_id="shard4") diff --git a/test/profiles.txt b/test/profiles.txt index 97ef13873..97691e4a1 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -104,11 +104,13 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgre test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgresql_psycopg2_nocextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_cextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4260 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_cextensions 4283 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_nocextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_nocextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_postgresql_psycopg2_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_postgresql_psycopg2_nocextensions 4266 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_sqlite_pysqlite_cextensions 4283 # TEST: test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove @@ -118,11 +120,13 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_postgresql_psycopg2_nocextensions 6426 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_cextensions 6426 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_nocextensions 6426 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_cextensions 6431 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_nocextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_cextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_nocextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_postgresql_psycopg2_cextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_postgresql_psycopg2_nocextensions 6428 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_sqlite_pysqlite_cextensions 6431 # TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline @@ -132,11 +136,13 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycop test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_nocextensions 40149 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 19280 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 28297 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_cextensions 20100 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_nocextensions 29138 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 32398 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 20289 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 37327 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_cextensions 20135 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_nocextensions 29138 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_sqlite_pysqlite_cextensions 20289 # TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols @@ -146,11 +152,13 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_nocextensions 30054 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 27144 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 30149 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_cextensions 26016 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_nocextensions 29068 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 32197 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 26127 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 31179 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_cextensions 26065 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_nocextensions 29068 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_sqlite_pysqlite_cextensions 26127 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity @@ -160,11 +168,13 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_postgresql_psycopg2_nocextensions 17988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_cextensions 17988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_nocextensions 17988 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_nocextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_nocextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_postgresql_psycopg2_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_postgresql_psycopg2_nocextensions 18988 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_sqlite_pysqlite_cextensions 18988 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity @@ -174,11 +184,13 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_postgresql_psycopg2_nocextensions 122553 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_cextensions 162315 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 165111 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_cextensions 119353 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_nocextensions 125352 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_cextensions 169566 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_nocextensions 171364 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_postgresql_psycopg2_cextensions 123602 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_postgresql_psycopg2_nocextensions 125352 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_sqlite_pysqlite_cextensions 161603 # TEST: test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks @@ -188,25 +200,29 @@ test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2. test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_postgresql_psycopg2_nocextensions 19219 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_cextensions 22288 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 22530 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_cextensions 18958 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_nocextensions 19492 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_cextensions 23067 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_nocextensions 23271 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_postgresql_psycopg2_cextensions 19228 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_postgresql_psycopg2_nocextensions 19480 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_sqlite_pysqlite_cextensions 21753 # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_load test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_mysql_mysqldb_cextensions 1411 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_mysql_mysqldb_nocextensions 1436 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_cextensions 1323 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_nocextensions 1348 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_cextensions 1249 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_nocextensions 1274 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_cextensions 1601 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1626 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1355 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1656 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1512 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_cextensions 1264 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1279 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1537 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_nocextensions 1671 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_cextensions 1340 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_nocextensions 1355 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_cextensions 1264 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_nocextensions 1279 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_sqlite_pysqlite_cextensions 1537 # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_no_load @@ -216,11 +232,13 @@ test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_nocextensions 117,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 117,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 117,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_nocextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_nocextensions 122,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_sqlite_pysqlite_cextensions 122,19 # TEST: test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect @@ -272,10 +290,13 @@ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_postgresql_psycopg2_nocextensions 45 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_nocextensions 45 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_mysql_mysqlconnector_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_nocextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_nocextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_mysql_mysqlconnector_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_mysql_mysqlconnector_nocextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_postgresql_psycopg2_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_postgresql_psycopg2_nocextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_sqlite_pysqlite_cextensions 43 @@ -289,10 +310,13 @@ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_nocextensions 80 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_nocextensions 80 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_mysql_mysqlconnector_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_nocextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_mysql_mysqlconnector_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_mysql_mysqlconnector_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_postgresql_psycopg2_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_postgresql_psycopg2_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_sqlite_pysqlite_cextensions 78 @@ -306,10 +330,13 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_postgresql_psycopg2_nocextensions 15 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_cextensions 15 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_nocextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_mysql_mysqlconnector_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_nocextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_nocextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_mysql_mysqlconnector_cextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_mysql_mysqlconnector_nocextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_postgresql_psycopg2_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_postgresql_psycopg2_nocextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_sqlite_pysqlite_cextensions 16 @@ -317,36 +344,42 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_string -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_cextensions 514 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_cextensions 451 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_nocextensions 15534 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_cextensions 20501 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_nocextensions 35521 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 457 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 394 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_nocextensions 15477 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 489 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_mysql_mysqlconnector_cextensions 109074 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 427 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 462 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 399 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_nocextensions 14462 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_cextensions 489 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_mysql_mysqlconnector_cextensions 53694 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_mysql_mysqlconnector_nocextensions 67694 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_cextensions 427 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_cextensions 462 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_cextensions 399 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_nocextensions 14462 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_unicode -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_cextensions 514 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_cextensions 451 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_nocextensions 45534 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_cextensions 20501 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_nocextensions 35521 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 457 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 394 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_nocextensions 15477 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 489 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_mysql_mysqlconnector_cextensions 109074 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 427 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 462 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 399 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_nocextensions 14462 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_cextensions 489 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_mysql_mysqlconnector_cextensions 53694 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_mysql_mysqlconnector_nocextensions 67694 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_cextensions 427 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_cextensions 462 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_cextensions 399 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_nocextensions 14462 # TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation -- cgit v1.2.1 From 6b9f62df10e1b1f557b9077613e5e96a08427460 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 8 Dec 2014 11:18:38 -0500 Subject: - force the _has_events flag to True on engines, so that profiling is more predictable - restore the profiling from before this change --- lib/sqlalchemy/testing/engines.py | 3 ++ test/profiles.txt | 75 +++++++++++---------------------------- 2 files changed, 24 insertions(+), 54 deletions(-) diff --git a/lib/sqlalchemy/testing/engines.py b/lib/sqlalchemy/testing/engines.py index 7d73e7423..444a79b70 100644 --- a/lib/sqlalchemy/testing/engines.py +++ b/lib/sqlalchemy/testing/engines.py @@ -215,6 +215,9 @@ def testing_engine(url=None, options=None): options = config.db_opts engine = create_engine(url, **options) + engine._has_events = True # enable event blocks, helps with + # profiling + if isinstance(engine.pool, pool.QueuePool): engine.pool._timeout = 0 engine.pool._max_overflow = 0 diff --git a/test/profiles.txt b/test/profiles.txt index 97691e4a1..97ef13873 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -104,13 +104,11 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgre test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgresql_psycopg2_nocextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_cextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4260 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_cextensions 4283 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_nocextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_nocextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_postgresql_psycopg2_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_postgresql_psycopg2_nocextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_sqlite_pysqlite_cextensions 4283 # TEST: test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove @@ -120,13 +118,11 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_postgresql_psycopg2_nocextensions 6426 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_cextensions 6426 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_nocextensions 6426 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_cextensions 6431 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_nocextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_cextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_nocextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_postgresql_psycopg2_cextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_postgresql_psycopg2_nocextensions 6428 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_sqlite_pysqlite_cextensions 6431 # TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline @@ -136,13 +132,11 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycop test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_nocextensions 40149 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 19280 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 28297 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_cextensions 20100 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_nocextensions 29138 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 20289 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 32398 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 37327 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_cextensions 20135 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_nocextensions 29138 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_sqlite_pysqlite_cextensions 20289 # TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols @@ -152,13 +146,11 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_nocextensions 30054 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 27144 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 30149 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_cextensions 26016 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_nocextensions 29068 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 26127 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 32197 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 31179 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_cextensions 26065 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_nocextensions 29068 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_sqlite_pysqlite_cextensions 26127 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity @@ -168,13 +160,11 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_postgresql_psycopg2_nocextensions 17988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_cextensions 17988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_nocextensions 17988 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_nocextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_nocextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_postgresql_psycopg2_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_postgresql_psycopg2_nocextensions 18988 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_sqlite_pysqlite_cextensions 18988 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity @@ -184,13 +174,11 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_postgresql_psycopg2_nocextensions 122553 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_cextensions 162315 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 165111 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_cextensions 119353 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_nocextensions 125352 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_cextensions 169566 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_nocextensions 171364 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_postgresql_psycopg2_cextensions 123602 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_postgresql_psycopg2_nocextensions 125352 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_sqlite_pysqlite_cextensions 161603 # TEST: test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks @@ -200,29 +188,25 @@ test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2. test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_postgresql_psycopg2_nocextensions 19219 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_cextensions 22288 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 22530 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_cextensions 18958 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_nocextensions 19492 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_cextensions 23067 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_nocextensions 23271 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_postgresql_psycopg2_cextensions 19228 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_postgresql_psycopg2_nocextensions 19480 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_sqlite_pysqlite_cextensions 21753 # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_load test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_mysql_mysqldb_cextensions 1411 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_mysql_mysqldb_nocextensions 1436 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_cextensions 1249 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_nocextensions 1274 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_cextensions 1323 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_nocextensions 1348 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_cextensions 1601 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1512 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_cextensions 1264 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1279 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1537 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1626 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1355 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1656 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_nocextensions 1671 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_cextensions 1264 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_nocextensions 1279 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_sqlite_pysqlite_cextensions 1537 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_cextensions 1340 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_nocextensions 1355 # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_no_load @@ -232,13 +216,11 @@ test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_nocextensions 117,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 117,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 117,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_nocextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_nocextensions 122,19 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_sqlite_pysqlite_cextensions 122,19 # TEST: test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect @@ -290,13 +272,10 @@ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_postgresql_psycopg2_nocextensions 45 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_nocextensions 45 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_mysql_mysqlconnector_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_nocextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_nocextensions 43 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_mysql_mysqlconnector_cextensions 43 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_mysql_mysqlconnector_nocextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_postgresql_psycopg2_cextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_postgresql_psycopg2_nocextensions 43 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_sqlite_pysqlite_cextensions 43 @@ -310,13 +289,10 @@ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_nocextensions 80 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_nocextensions 80 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_mysql_mysqlconnector_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_nocextensions 78 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_mysql_mysqlconnector_cextensions 78 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_mysql_mysqlconnector_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_postgresql_psycopg2_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_postgresql_psycopg2_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_sqlite_pysqlite_cextensions 78 @@ -330,13 +306,10 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_postgresql_psycopg2_nocextensions 15 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_cextensions 15 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_nocextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_mysql_mysqlconnector_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_nocextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_nocextensions 16 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_mysql_mysqlconnector_cextensions 16 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_mysql_mysqlconnector_nocextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_postgresql_psycopg2_cextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_postgresql_psycopg2_nocextensions 16 test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_sqlite_pysqlite_cextensions 16 @@ -344,42 +317,36 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_string -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_cextensions 451 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_cextensions 514 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_nocextensions 15534 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_cextensions 20501 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_nocextensions 35521 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 394 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 457 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_nocextensions 15477 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_mysql_mysqlconnector_cextensions 109074 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 427 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 489 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 399 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 462 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_nocextensions 14462 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_mysql_mysqlconnector_cextensions 53694 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_mysql_mysqlconnector_nocextensions 67694 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_cextensions 427 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_cextensions 489 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_cextensions 399 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_cextensions 462 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_nocextensions 14462 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_unicode -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_cextensions 451 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_cextensions 514 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_nocextensions 45534 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_cextensions 20501 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_nocextensions 35521 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 394 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 457 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_nocextensions 15477 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_mysql_mysqlconnector_cextensions 109074 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 427 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 489 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 399 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 462 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_nocextensions 14462 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_mysql_mysqlconnector_cextensions 53694 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_mysql_mysqlconnector_nocextensions 67694 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_cextensions 427 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_cextensions 489 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_nocextensions 14489 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_cextensions 399 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_cextensions 462 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_nocextensions 14462 # TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation -- cgit v1.2.1 From 8553c195c24f67ff5d75893ddad57d1003fb9759 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 8 Dec 2014 12:34:40 -0500 Subject: - autoinc here for oracle --- test/orm/test_naturalpks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/orm/test_naturalpks.py b/test/orm/test_naturalpks.py index 709e1c0b1..60387ddce 100644 --- a/test/orm/test_naturalpks.py +++ b/test/orm/test_naturalpks.py @@ -1228,7 +1228,9 @@ class JoinedInheritancePKOnFKTest(fixtures.MappedTest): Table( 'engineer', metadata, - Column('id', Integer, primary_key=True), + Column( + 'id', Integer, + primary_key=True, test_needs_autoincrement=True), Column( 'person_name', String(50), ForeignKey('person.name', **fk_args)), -- cgit v1.2.1 From b7cf11b163dd7d15f56634a41dcceb880821ecf3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 8 Dec 2014 14:05:20 -0500 Subject: - simplify the "noconnection" error handling, setting _handle_dbapi_exception_noconnection() to only invoke in the case of raw_connection() in the constructor of Connection. in all other cases the Connection proceeds with _handle_dbapi_exception() including revalidate. --- lib/sqlalchemy/engine/base.py | 36 +++++++++++++++++++----------------- lib/sqlalchemy/engine/threadlocal.py | 2 +- test/engine/test_reconnect.py | 4 ++-- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 23348469d..dd8ea275c 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -265,18 +265,20 @@ class Connection(Connectable): try: return self.__connection except AttributeError: - return self._revalidate_connection(_wrap=True) + try: + return self._revalidate_connection() + except Exception as e: + self._handle_dbapi_exception(e, None, None, None, None) - def _revalidate_connection(self, _wrap): + def _revalidate_connection(self): if self.__branch_from: - return self.__branch_from._revalidate_connection(_wrap=_wrap) + return self.__branch_from._revalidate_connection() if self.__can_reconnect and self.__invalid: if self.__transaction is not None: raise exc.InvalidRequestError( "Can't reconnect until invalid " "transaction is rolled back") - self.__connection = self.engine.raw_connection( - _connection=self, _wrap=_wrap) + self.__connection = self.engine.raw_connection(_connection=self) self.__invalid = False return self.__connection raise exc.ResourceClosedError("This Connection is closed") @@ -817,7 +819,7 @@ class Connection(Connectable): try: conn = self.__connection except AttributeError: - conn = self._revalidate_connection(_wrap=False) + conn = self._revalidate_connection() dialect = self.dialect ctx = dialect.execution_ctx_cls._init_default( @@ -955,7 +957,7 @@ class Connection(Connectable): try: conn = self.__connection except AttributeError: - conn = self._revalidate_connection(_wrap=False) + conn = self._revalidate_connection() context = constructor(dialect, self, conn, *args) except Exception as e: @@ -1248,8 +1250,7 @@ class Connection(Connectable): self.close() @classmethod - def _handle_dbapi_exception_noconnection( - cls, e, dialect, engine, connection): + def _handle_dbapi_exception_noconnection(cls, e, dialect, engine): exc_info = sys.exc_info() @@ -1271,7 +1272,7 @@ class Connection(Connectable): if engine._has_events: ctx = ExceptionContextImpl( - e, sqlalchemy_exception, engine, connection, None, None, + e, sqlalchemy_exception, engine, None, None, None, None, None, is_disconnect) for fn in engine.dispatch.handle_error: try: @@ -1957,17 +1958,18 @@ class Engine(Connectable, log.Identified): """ return self.run_callable(self.dialect.has_table, table_name, schema) - def _wrap_pool_connect(self, fn, connection, wrap=True): - if not wrap: - return fn() + def _wrap_pool_connect(self, fn, connection): dialect = self.dialect try: return fn() except dialect.dbapi.Error as e: - Connection._handle_dbapi_exception_noconnection( - e, dialect, self, connection) + if connection is None: + Connection._handle_dbapi_exception_noconnection( + e, dialect, self) + else: + util.reraise(*sys.exc_info()) - def raw_connection(self, _connection=None, _wrap=True): + def raw_connection(self, _connection=None): """Return a "raw" DBAPI connection from the connection pool. The returned object is a proxied version of the DBAPI @@ -1984,7 +1986,7 @@ class Engine(Connectable, log.Identified): """ return self._wrap_pool_connect( - self.pool.unique_connection, _connection, _wrap) + self.pool.unique_connection, _connection) class OptionEngine(Engine): diff --git a/lib/sqlalchemy/engine/threadlocal.py b/lib/sqlalchemy/engine/threadlocal.py index 824b68fdf..e64ab09f4 100644 --- a/lib/sqlalchemy/engine/threadlocal.py +++ b/lib/sqlalchemy/engine/threadlocal.py @@ -61,7 +61,7 @@ class TLEngine(base.Engine): connection = self._tl_connection_cls( self, self._wrap_pool_connect( - self.pool.connect, connection, wrap=True), + self.pool.connect, connection), **kw) self._connections.conn = weakref.ref(connection) diff --git a/test/engine/test_reconnect.py b/test/engine/test_reconnect.py index 0efce87ce..4500ada6a 100644 --- a/test/engine/test_reconnect.py +++ b/test/engine/test_reconnect.py @@ -517,7 +517,7 @@ class RealReconnectTest(fixtures.TestBase): assert c1.invalidated assert c1_branch.invalidated - c1_branch._revalidate_connection(_wrap=True) + c1_branch._revalidate_connection() assert not c1.invalidated assert not c1_branch.invalidated @@ -535,7 +535,7 @@ class RealReconnectTest(fixtures.TestBase): assert c1.invalidated assert c1_branch.invalidated - c1._revalidate_connection(_wrap=True) + c1._revalidate_connection() assert not c1.invalidated assert not c1_branch.invalidated -- cgit v1.2.1 From 06738f665ea936246a3813ad7de01e98ff8d519a Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 8 Dec 2014 15:15:02 -0500 Subject: - identify another spot where _handle_dbapi_error() needs to do something differently for the case where it is called in an already-invalidated state; don't call upon self.connection --- lib/sqlalchemy/engine/base.py | 7 ++++--- test/engine/test_parseconnect.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index dd8ea275c..9a8610344 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -1243,9 +1243,10 @@ class Connection(Connectable): del self._reentrant_error if self._is_disconnect: del self._is_disconnect - dbapi_conn_wrapper = self.connection - self.engine.pool._invalidate(dbapi_conn_wrapper, e) - self.invalidate(e) + if not self.invalidated: + dbapi_conn_wrapper = self.__connection + self.engine.pool._invalidate(dbapi_conn_wrapper, e) + self.invalidate(e) if self.should_close_with_result: self.close() diff --git a/test/engine/test_parseconnect.py b/test/engine/test_parseconnect.py index 4a3da7d1c..8d659420d 100644 --- a/test/engine/test_parseconnect.py +++ b/test/engine/test_parseconnect.py @@ -396,6 +396,34 @@ class CreateEngineTest(fixtures.TestBase): except tsa.exc.DBAPIError as de: assert not de.connection_invalidated + @testing.requires.sqlite + def test_cant_connect_stay_invalidated(self): + e = create_engine('sqlite://') + sqlite3 = e.dialect.dbapi + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://') + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.is_disconnect + + conn = eng.connect() + + conn.invalidate() + + eng.pool._creator = Mock( + side_effect=sqlite3.ProgrammingError( + "Cannot operate on a closed database.")) + + try: + conn.connection + assert False + except tsa.exc.DBAPIError: + assert conn.invalidated + @testing.requires.sqlite def test_dont_touch_non_dbapi_exception_on_connect(self): e = create_engine('sqlite://') -- cgit v1.2.1 From c86c593ec3b913361999a1970efae3e6f3d831fa Mon Sep 17 00:00:00 2001 From: Yuval Langer Date: Tue, 9 Dec 2014 04:19:18 +0200 Subject: Removing unneeded space. --- doc/build/core/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/core/tutorial.rst b/doc/build/core/tutorial.rst index b6c07bdc0..e96217f79 100644 --- a/doc/build/core/tutorial.rst +++ b/doc/build/core/tutorial.rst @@ -370,7 +370,7 @@ Selecting ========== We began with inserts just so that our test database had some data in it. The -more interesting part of the data is selecting it ! We'll cover UPDATE and +more interesting part of the data is selecting it! We'll cover UPDATE and DELETE statements later. The primary construct used to generate SELECT statements is the :func:`.select` function: -- cgit v1.2.1 From eee617e08eb761de7279de31246d904ca6b17da7 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 10 Dec 2014 12:11:59 -0500 Subject: - rework the handle error on connect tests from test_parsconnect where they don't really belong into a new suite in test_execute --- test/engine/test_execute.py | 245 +++++++++++++++++++++++++++++++++++++++ test/engine/test_parseconnect.py | 236 ------------------------------------- 2 files changed, 245 insertions(+), 236 deletions(-) diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index 5c3279ba9..8e58d202d 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -1901,6 +1901,251 @@ class HandleErrorTest(fixtures.TestBase): self._test_alter_disconnect(False, False) +class HandleInvalidatedOnConnectTest(fixtures.TestBase): + __requires__ = ('sqlite', ) + + def setUp(self): + e = create_engine('sqlite://') + + connection = Mock( + get_server_version_info=Mock(return_value='5.0')) + + def connect(*args, **kwargs): + return connection + dbapi = Mock( + sqlite_version_info=(99, 9, 9,), + version_info=(99, 9, 9,), + sqlite_version='99.9.9', + paramstyle='named', + connect=Mock(side_effect=connect) + ) + + sqlite3 = e.dialect.dbapi + dbapi.Error = sqlite3.Error, + dbapi.ProgrammingError = sqlite3.ProgrammingError + + self.dbapi = dbapi + self.ProgrammingError = sqlite3.ProgrammingError + + def test_wraps_connect_in_dbapi(self): + dbapi = self.dbapi + dbapi.connect = Mock( + side_effect=self.ProgrammingError("random error")) + try: + create_engine('sqlite://', module=dbapi).connect() + assert False + except tsa.exc.DBAPIError as de: + assert not de.connection_invalidated + + def test_handle_error_event_connect(self): + dbapi = self.dbapi + dbapi.connect = Mock( + side_effect=self.ProgrammingError("random error")) + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is None + raise MySpecialException("failed operation") + + assert_raises( + MySpecialException, + eng.connect + ) + + def test_handle_error_event_revalidate(self): + dbapi = self.dbapi + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi, _initialize=False) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is conn + assert isinstance(ctx.sqlalchemy_exception, tsa.exc.ProgrammingError) + raise MySpecialException("failed operation") + + conn = eng.connect() + conn.invalidate() + + dbapi.connect = Mock( + side_effect=self.ProgrammingError("random error")) + + assert_raises( + MySpecialException, + getattr, conn, 'connection' + ) + + def test_handle_error_event_implicit_revalidate(self): + dbapi = self.dbapi + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi, _initialize=False) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is conn + assert isinstance( + ctx.sqlalchemy_exception, tsa.exc.ProgrammingError) + raise MySpecialException("failed operation") + + conn = eng.connect() + conn.invalidate() + + dbapi.connect = Mock( + side_effect=self.ProgrammingError("random error")) + + assert_raises( + MySpecialException, + conn.execute, select([1]) + ) + + def test_handle_error_custom_connect(self): + dbapi = self.dbapi + + class MySpecialException(Exception): + pass + + def custom_connect(): + raise self.ProgrammingError("random error") + + eng = create_engine('sqlite://', module=dbapi, creator=custom_connect) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.engine is eng + assert ctx.connection is None + raise MySpecialException("failed operation") + + assert_raises( + MySpecialException, + eng.connect + ) + + def test_handle_error_event_connect_invalidate_flag(self): + dbapi = self.dbapi + dbapi.connect = Mock( + side_effect=self.ProgrammingError( + "Cannot operate on a closed database.")) + + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://', module=dbapi) + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.is_disconnect + ctx.is_disconnect = False + + try: + eng.connect() + assert False + except tsa.exc.DBAPIError as de: + assert not de.connection_invalidated + + def test_cant_connect_stay_invalidated(self): + class MySpecialException(Exception): + pass + + eng = create_engine('sqlite://') + + @event.listens_for(eng, "handle_error") + def handle_error(ctx): + assert ctx.is_disconnect + + conn = eng.connect() + + conn.invalidate() + + eng.pool._creator = Mock( + side_effect=self.ProgrammingError( + "Cannot operate on a closed database.")) + + try: + conn.connection + assert False + except tsa.exc.DBAPIError: + assert conn.invalidated + + def _test_dont_touch_non_dbapi_exception_on_connect(self, connect_fn): + dbapi = self.dbapi + dbapi.connect = Mock(side_effect=TypeError("I'm not a DBAPI error")) + + e = create_engine('sqlite://', module=dbapi) + e.dialect.is_disconnect = is_disconnect = Mock() + assert_raises_message( + TypeError, + "I'm not a DBAPI error", + connect_fn, e + ) + eq_(is_disconnect.call_count, 0) + + def test_dont_touch_non_dbapi_exception_on_connect(self): + self._test_dont_touch_non_dbapi_exception_on_connect( + lambda engine: engine.connect()) + + def test_dont_touch_non_dbapi_exception_on_contextual_connect(self): + self._test_dont_touch_non_dbapi_exception_on_connect( + lambda engine: engine.contextual_connect()) + + def test_ensure_dialect_does_is_disconnect_no_conn(self): + """test that is_disconnect() doesn't choke if no connection, + cursor given.""" + dialect = testing.db.dialect + dbapi = dialect.dbapi + assert not dialect.is_disconnect( + dbapi.OperationalError("test"), None, None) + + def _test_invalidate_on_connect(self, connect_fn): + """test that is_disconnect() is called during connect. + + interpretation of connection failures are not supported by + every backend. + + """ + + dbapi = self.dbapi + dbapi.connect = Mock( + side_effect=self.ProgrammingError( + "Cannot operate on a closed database.")) + try: + connect_fn(create_engine('sqlite://', module=dbapi)) + assert False + except tsa.exc.DBAPIError as de: + assert de.connection_invalidated + + def test_invalidate_on_connect(self): + """test that is_disconnect() is called during connect. + + interpretation of connection failures are not supported by + every backend. + + """ + self._test_invalidate_on_connect(lambda engine: engine.connect()) + + def test_invalidate_on_contextual_connect(self): + """test that is_disconnect() is called during connect. + + interpretation of connection failures are not supported by + every backend. + + """ + self._test_invalidate_on_connect( + lambda engine: engine.contextual_connect()) + + class ProxyConnectionTest(fixtures.TestBase): """These are the same tests as EngineEventsTest, except using diff --git a/test/engine/test_parseconnect.py b/test/engine/test_parseconnect.py index 8d659420d..e53a99e15 100644 --- a/test/engine/test_parseconnect.py +++ b/test/engine/test_parseconnect.py @@ -238,242 +238,6 @@ class CreateEngineTest(fixtures.TestBase): assert_raises(TypeError, create_engine, 'mysql+mysqldb://', use_unicode=True, module=mock_dbapi) - @testing.requires.sqlite - def test_wraps_connect_in_dbapi(self): - e = create_engine('sqlite://') - sqlite3 = e.dialect.dbapi - dbapi = MockDBAPI() - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock( - side_effect=sqlite3.ProgrammingError("random error")) - try: - create_engine('sqlite://', module=dbapi).connect() - assert False - except tsa.exc.DBAPIError as de: - assert not de.connection_invalidated - - @testing.requires.sqlite - def test_handle_error_event_connect(self): - e = create_engine('sqlite://') - dbapi = MockDBAPI() - sqlite3 = e.dialect.dbapi - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock( - side_effect=sqlite3.ProgrammingError("random error")) - - class MySpecialException(Exception): - pass - - eng = create_engine('sqlite://', module=dbapi) - - @event.listens_for(eng, "handle_error") - def handle_error(ctx): - assert ctx.engine is eng - assert ctx.connection is None - raise MySpecialException("failed operation") - - assert_raises( - MySpecialException, - eng.connect - ) - - @testing.requires.sqlite - def test_handle_error_event_revalidate(self): - e = create_engine('sqlite://') - dbapi = MockDBAPI() - sqlite3 = e.dialect.dbapi - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - - class MySpecialException(Exception): - pass - - eng = create_engine('sqlite://', module=dbapi, _initialize=False) - - @event.listens_for(eng, "handle_error") - def handle_error(ctx): - assert ctx.engine is eng - assert ctx.connection is conn - assert isinstance(ctx.sqlalchemy_exception, exc.ProgrammingError) - raise MySpecialException("failed operation") - - conn = eng.connect() - conn.invalidate() - - dbapi.connect = Mock( - side_effect=sqlite3.ProgrammingError("random error")) - - assert_raises( - MySpecialException, - getattr, conn, 'connection' - ) - - @testing.requires.sqlite - def test_handle_error_event_implicit_revalidate(self): - e = create_engine('sqlite://') - dbapi = MockDBAPI() - sqlite3 = e.dialect.dbapi - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - - class MySpecialException(Exception): - pass - - eng = create_engine('sqlite://', module=dbapi, _initialize=False) - - @event.listens_for(eng, "handle_error") - def handle_error(ctx): - assert ctx.engine is eng - assert ctx.connection is conn - assert isinstance(ctx.sqlalchemy_exception, exc.ProgrammingError) - raise MySpecialException("failed operation") - - conn = eng.connect() - conn.invalidate() - - dbapi.connect = Mock( - side_effect=sqlite3.ProgrammingError("random error")) - - assert_raises( - MySpecialException, - conn.execute, select([1]) - ) - - @testing.requires.sqlite - def test_handle_error_custom_connect(self): - e = create_engine('sqlite://') - - dbapi = MockDBAPI() - sqlite3 = e.dialect.dbapi - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - - class MySpecialException(Exception): - pass - - def custom_connect(): - raise sqlite3.ProgrammingError("random error") - - eng = create_engine('sqlite://', module=dbapi, creator=custom_connect) - - @event.listens_for(eng, "handle_error") - def handle_error(ctx): - assert ctx.engine is eng - assert ctx.connection is None - raise MySpecialException("failed operation") - - assert_raises( - MySpecialException, - eng.connect - ) - - @testing.requires.sqlite - def test_handle_error_event_connect_invalidate_flag(self): - e = create_engine('sqlite://') - dbapi = MockDBAPI() - sqlite3 = e.dialect.dbapi - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock( - side_effect=sqlite3.ProgrammingError( - "Cannot operate on a closed database.")) - - class MySpecialException(Exception): - pass - - eng = create_engine('sqlite://', module=dbapi) - - @event.listens_for(eng, "handle_error") - def handle_error(ctx): - assert ctx.is_disconnect - ctx.is_disconnect = False - - try: - eng.connect() - assert False - except tsa.exc.DBAPIError as de: - assert not de.connection_invalidated - - @testing.requires.sqlite - def test_cant_connect_stay_invalidated(self): - e = create_engine('sqlite://') - sqlite3 = e.dialect.dbapi - - class MySpecialException(Exception): - pass - - eng = create_engine('sqlite://') - - @event.listens_for(eng, "handle_error") - def handle_error(ctx): - assert ctx.is_disconnect - - conn = eng.connect() - - conn.invalidate() - - eng.pool._creator = Mock( - side_effect=sqlite3.ProgrammingError( - "Cannot operate on a closed database.")) - - try: - conn.connection - assert False - except tsa.exc.DBAPIError: - assert conn.invalidated - - @testing.requires.sqlite - def test_dont_touch_non_dbapi_exception_on_connect(self): - e = create_engine('sqlite://') - sqlite3 = e.dialect.dbapi - - dbapi = MockDBAPI() - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock(side_effect=TypeError("I'm not a DBAPI error")) - e = create_engine('sqlite://', module=dbapi) - e.dialect.is_disconnect = is_disconnect = Mock() - assert_raises_message( - TypeError, - "I'm not a DBAPI error", - e.connect - ) - eq_(is_disconnect.call_count, 0) - - def test_ensure_dialect_does_is_disconnect_no_conn(self): - """test that is_disconnect() doesn't choke if no connection, - cursor given.""" - dialect = testing.db.dialect - dbapi = dialect.dbapi - assert not dialect.is_disconnect( - dbapi.OperationalError("test"), None, None) - - @testing.requires.sqlite - def test_invalidate_on_connect(self): - """test that is_disconnect() is called during connect. - - interpretation of connection failures are not supported by - every backend. - - """ - - e = create_engine('sqlite://') - sqlite3 = e.dialect.dbapi - - dbapi = MockDBAPI() - dbapi.Error = sqlite3.Error, - dbapi.ProgrammingError = sqlite3.ProgrammingError - dbapi.connect = Mock( - side_effect=sqlite3.ProgrammingError( - "Cannot operate on a closed database.")) - try: - create_engine('sqlite://', module=dbapi).connect() - assert False - except tsa.exc.DBAPIError as de: - assert de.connection_invalidated - def test_urlattr(self): """test the url attribute on ``Engine``.""" -- cgit v1.2.1 From 347db81aea9bfe301a9fe1fade644ad099545f3e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 10 Dec 2014 12:15:14 -0500 Subject: - keep working on fixing #3266, more cases, more tests --- lib/sqlalchemy/engine/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 9a8610344..918ee0e37 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -1926,10 +1926,11 @@ class Engine(Connectable, log.Identified): """ - return self._connection_cls(self, - self.pool.connect(), - close_with_result=close_with_result, - **kwargs) + return self._connection_cls( + self, + self._wrap_pool_connect(self.pool.connect, None), + close_with_result=close_with_result, + **kwargs) def table_names(self, schema=None, connection=None): """Return a list of all table names available in the database. -- cgit v1.2.1 From 3c70f609507ccc6775495cc533265aeb645528cd Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 10 Dec 2014 13:08:53 -0500 Subject: - fix up query update /delete documentation, make warnings a lot clearer, partial fixes for #3252 --- lib/sqlalchemy/orm/query.py | 179 +++++++++++++++++++++++++++----------------- 1 file changed, 110 insertions(+), 69 deletions(-) diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 9b7747e15..1afffb90e 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -2725,6 +2725,18 @@ class Query(object): Deletes rows matched by this query from the database. + E.g.:: + + sess.query(User).filter(User.age == 25).\\ + delete(synchronize_session=False) + + sess.query(User).filter(User.age == 25).\\ + delete(synchronize_session='evaluate') + + .. warning:: The :meth:`.Query.delete` method is a "bulk" operation, + which bypasses ORM unit-of-work automation in favor of greater + performance. **Please read all caveats and warnings below.** + :param synchronize_session: chooses the strategy for the removal of matched objects from the session. Valid values are: @@ -2743,8 +2755,7 @@ class Query(object): ``'evaluate'`` - Evaluate the query's criteria in Python straight on the objects in the session. If evaluation of the criteria isn't - implemented, an error is raised. In that case you probably - want to use the 'fetch' strategy as a fallback. + implemented, an error is raised. The expression evaluator currently doesn't account for differing string collations between the database and Python. @@ -2752,29 +2763,42 @@ class Query(object): :return: the count of rows matched as returned by the database's "row count" feature. - This method has several key caveats: - - * The method does **not** offer in-Python cascading of relationships - - it is assumed that ON DELETE CASCADE/SET NULL/etc. is configured - for any foreign key references which require it, otherwise the - database may emit an integrity violation if foreign key references - are being enforced. - - After the DELETE, dependent objects in the :class:`.Session` which - were impacted by an ON DELETE may not contain the current - state, or may have been deleted. This issue is resolved once the - :class:`.Session` is expired, - which normally occurs upon :meth:`.Session.commit` or can be forced - by using :meth:`.Session.expire_all`. Accessing an expired object - whose row has been deleted will invoke a SELECT to locate the - row; when the row is not found, an - :class:`~sqlalchemy.orm.exc.ObjectDeletedError` is raised. - - * The :meth:`.MapperEvents.before_delete` and - :meth:`.MapperEvents.after_delete` - events are **not** invoked from this method. Instead, the - :meth:`.SessionEvents.after_bulk_delete` method is provided to act - upon a mass DELETE of entity rows. + .. warning:: **Additional Caveats for bulk query deletes** + + * The method does **not** offer in-Python cascading of + relationships - it is assumed that ON DELETE CASCADE/SET + NULL/etc. is configured for any foreign key references + which require it, otherwise the database may emit an + integrity violation if foreign key references are being + enforced. + + After the DELETE, dependent objects in the + :class:`.Session` which were impacted by an ON DELETE + may not contain the current state, or may have been + deleted. This issue is resolved once the + :class:`.Session` is expired, which normally occurs upon + :meth:`.Session.commit` or can be forced by using + :meth:`.Session.expire_all`. Accessing an expired + object whose row has been deleted will invoke a SELECT + to locate the row; when the row is not found, an + :class:`~sqlalchemy.orm.exc.ObjectDeletedError` is + raised. + + * The ``'fetch'`` strategy results in an additional + SELECT statement emitted and will significantly reduce + performance. + + * The ``'evaulate'`` strategy performs a scan of + all matching objects within the :class:`.Session`; if the + contents of the :class:`.Session` are expired, such as + via a proceeding :meth:`.Session.commit` call, **this will + result in SELECT queries emitted for every matching object**. + + * The :meth:`.MapperEvents.before_delete` and + :meth:`.MapperEvents.after_delete` + events **are not invoked** from this method. Instead, the + :meth:`.SessionEvents.after_bulk_delete` method is provided to + act upon a mass DELETE of entity rows. .. seealso:: @@ -2797,17 +2821,21 @@ class Query(object): E.g.:: - sess.query(User).filter(User.age == 25).\ - update({User.age: User.age - 10}, synchronize_session='fetch') + sess.query(User).filter(User.age == 25).\\ + update({User.age: User.age - 10}, synchronize_session=False) - - sess.query(User).filter(User.age == 25).\ + sess.query(User).filter(User.age == 25).\\ update({"age": User.age - 10}, synchronize_session='evaluate') + .. warning:: The :meth:`.Query.update` method is a "bulk" operation, + which bypasses ORM unit-of-work automation in favor of greater + performance. **Please read all caveats and warnings below.** + + :param values: a dictionary with attributes names, or alternatively - mapped attributes or SQL expressions, as keys, and literal - values or sql expressions as values. + mapped attributes or SQL expressions, as keys, and literal + values or sql expressions as values. .. versionchanged:: 1.0.0 - string names in the values dictionary are now resolved against the mapped entity; previously, these @@ -2815,7 +2843,7 @@ class Query(object): translation. :param synchronize_session: chooses the strategy to update the - attributes on objects in the session. Valid values are: + attributes on objects in the session. Valid values are: ``False`` - don't synchronize the session. This option is the most efficient and is reliable once the session is expired, which @@ -2836,43 +2864,56 @@ class Query(object): string collations between the database and Python. :return: the count of rows matched as returned by the database's - "row count" feature. - - This method has several key caveats: - - * The method does **not** offer in-Python cascading of relationships - - it is assumed that ON UPDATE CASCADE is configured for any foreign - key references which require it, otherwise the database may emit an - integrity violation if foreign key references are being enforced. - - After the UPDATE, dependent objects in the :class:`.Session` which - were impacted by an ON UPDATE CASCADE may not contain the current - state; this issue is resolved once the :class:`.Session` is expired, - which normally occurs upon :meth:`.Session.commit` or can be forced - by using :meth:`.Session.expire_all`. - - * The method supports multiple table updates, as - detailed in :ref:`multi_table_updates`, and this behavior does - extend to support updates of joined-inheritance and other multiple - table mappings. However, the **join condition of an inheritance - mapper is currently not automatically rendered**. - Care must be taken in any multiple-table update to explicitly - include the joining condition between those tables, even in mappings - where this is normally automatic. - E.g. if a class ``Engineer`` subclasses ``Employee``, an UPDATE of - the ``Engineer`` local table using criteria against the ``Employee`` - local table might look like:: - - session.query(Engineer).\\ - filter(Engineer.id == Employee.id).\\ - filter(Employee.name == 'dilbert').\\ - update({"engineer_type": "programmer"}) - - * The :meth:`.MapperEvents.before_update` and - :meth:`.MapperEvents.after_update` - events are **not** invoked from this method. Instead, the - :meth:`.SessionEvents.after_bulk_update` method is provided to act - upon a mass UPDATE of entity rows. + "row count" feature. + + .. warning:: **Additional Caveats for bulk query updates** + + * The method does **not** offer in-Python cascading of + relationships - it is assumed that ON UPDATE CASCADE is + configured for any foreign key references which require + it, otherwise the database may emit an integrity + violation if foreign key references are being enforced. + + After the UPDATE, dependent objects in the + :class:`.Session` which were impacted by an ON UPDATE + CASCADE may not contain the current state; this issue is + resolved once the :class:`.Session` is expired, which + normally occurs upon :meth:`.Session.commit` or can be + forced by using :meth:`.Session.expire_all`. + + * The ``'fetch'`` strategy results in an additional + SELECT statement emitted and will significantly reduce + performance. + + * The ``'evaulate'`` strategy performs a scan of + all matching objects within the :class:`.Session`; if the + contents of the :class:`.Session` are expired, such as + via a proceeding :meth:`.Session.commit` call, **this will + result in SELECT queries emitted for every matching object**. + + * The method supports multiple table updates, as detailed + in :ref:`multi_table_updates`, and this behavior does + extend to support updates of joined-inheritance and + other multiple table mappings. However, the **join + condition of an inheritance mapper is not + automatically rendered**. Care must be taken in any + multiple-table update to explicitly include the joining + condition between those tables, even in mappings where + this is normally automatic. E.g. if a class ``Engineer`` + subclasses ``Employee``, an UPDATE of the ``Engineer`` + local table using criteria against the ``Employee`` + local table might look like:: + + session.query(Engineer).\\ + filter(Engineer.id == Employee.id).\\ + filter(Employee.name == 'dilbert').\\ + update({"engineer_type": "programmer"}) + + * The :meth:`.MapperEvents.before_update` and + :meth:`.MapperEvents.after_update` + events **are not invoked from this method**. Instead, the + :meth:`.SessionEvents.after_bulk_update` method is provided to + act upon a mass UPDATE of entity rows. .. seealso:: -- cgit v1.2.1 From 08e02579e03bf37cfc742c549b837841ec8f7ffe Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 12 Dec 2014 15:55:34 -0500 Subject: - update zoomark --- test/profiles.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/profiles.txt b/test/profiles.txt index 97ef13873..c11000e29 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -351,12 +351,12 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite # TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5562,277,3697,11893,1106,1968,2433 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5606,277,3929,13595,1223,2011,2692 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 5238,273,3577,11529,1077,1886,2439 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 5260,273,3673,12701,1171,1893,2631 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_cextensions 5221,273,3577,11529,1077,1883,2439 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_nocextensions 5243,273,3697,12796,1187,1923,2653 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5562,292,3697,11893,1106,1968,2433 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5606,292,3929,13595,1223,2011,2692 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 5497,274,3609,11647,1097,1921,2486 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 5519,274,3705,12819,1191,1928,2678 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_cextensions 5497,273,3577,11529,1077,1883,2439 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_nocextensions 5519,273,3697,12796,1187,1923,2653 # TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation -- cgit v1.2.1 From cf7981f60d485f17465f44c6ff651ae283ade377 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 12 Dec 2014 19:59:11 -0500 Subject: - Added new method :meth:`.Session.invalidate`, functions similarly to :meth:`.Session.close`, except also calls :meth:`.Connection.invalidate` on all connections, guaranteeing that they will not be returned to the connection pool. This is useful in situations e.g. dealing with gevent timeouts when it is not safe to use the connection further, even for rollbacks. references #3258 --- doc/build/changelog/changelog_09.rst | 12 +++++++++++ lib/sqlalchemy/orm/session.py | 42 ++++++++++++++++++++++++++++++++++-- test/orm/test_session.py | 3 +++ test/orm/test_transaction.py | 17 +++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index f10d48273..419827959 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,18 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: enhancement, orm + :versions: 1.0.0 + + Added new method :meth:`.Session.invalidate`, functions similarly + to :meth:`.Session.close`, except also calls + :meth:`.Connection.invalidate` + on all connections, guaranteeing that they will not be returned to + the connection pool. This is useful in situations e.g. dealing + with gevent timeouts when it is not safe to use the connection further, + even for rollbacks. + .. change:: :tags: bug, examples :versions: 1.0.0 diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index d40d28154..507e99b2e 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -435,11 +435,13 @@ class SessionTransaction(object): self.session.dispatch.after_rollback(self.session) - def close(self): + def close(self, invalidate=False): self.session.transaction = self._parent if self._parent is None: for connection, transaction, autoclose in \ set(self._connections.values()): + if invalidate: + connection.invalidate() if autoclose: connection.close() else: @@ -1000,10 +1002,46 @@ class Session(_SessionClassMethods): not use any connection resources until they are first needed. """ + self._close_impl(invalidate=False) + + def invalidate(self): + """Close this Session, using connection invalidation. + + This is a variant of :meth:`.Session.close` that will additionally + ensure that the :meth:`.Connection.invalidate` method will be called + on all :class:`.Connection` objects. This can be called when + the database is known to be in a state where the connections are + no longer safe to be used. + + E.g.:: + + try: + sess = Session() + sess.add(User()) + sess.commit() + except gevent.Timeout: + sess.invalidate() + raise + except: + sess.rollback() + raise + + This clears all items and ends any transaction in progress. + + If this session were created with ``autocommit=False``, a new + transaction is immediately begun. Note that this new transaction does + not use any connection resources until they are first needed. + + .. versionadded:: 0.9.9 + + """ + self._close_impl(invalidate=True) + + def _close_impl(self, invalidate): self.expunge_all() if self.transaction is not None: for transaction in self.transaction._iterate_parents(): - transaction.close() + transaction.close(invalidate) def expunge_all(self): """Remove all object instances from this ``Session``. diff --git a/test/orm/test_session.py b/test/orm/test_session.py index b81c03f88..2aa0cd3eb 100644 --- a/test/orm/test_session.py +++ b/test/orm/test_session.py @@ -1364,6 +1364,9 @@ class DisposedStates(fixtures.MappedTest): def test_close(self): self._test_session().close() + def test_invalidate(self): + self._test_session().invalidate() + def test_expunge_all(self): self._test_session().expunge_all() diff --git a/test/orm/test_transaction.py b/test/orm/test_transaction.py index ba31e4c7d..1d7e8e693 100644 --- a/test/orm/test_transaction.py +++ b/test/orm/test_transaction.py @@ -184,6 +184,23 @@ class SessionTransactionTest(FixtureTest): assert users.count().scalar() == 1 assert addresses.count().scalar() == 1 + @testing.requires.independent_connections + def test_invalidate(self): + User, users = self.classes.User, self.tables.users + mapper(User, users) + sess = Session() + u = User(name='u1') + sess.add(u) + sess.flush() + c1 = sess.connection(User) + + sess.invalidate() + assert c1.invalidated + + eq_(sess.query(User).all(), []) + c2 = sess.connection(User) + assert not c2.invalidated + def test_subtransaction_on_noautocommit(self): User, users = self.classes.User, self.tables.users -- cgit v1.2.1 From 91af7337878612b2497269e600eef147a0f5bb30 Mon Sep 17 00:00:00 2001 From: Jon Nelson Date: Tue, 11 Nov 2014 22:46:07 -0600 Subject: - fix unique constraint parsing for sqlite -- may return '' for name, however --- lib/sqlalchemy/dialects/sqlite/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index ccd7f2539..30d8a6ea3 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -1173,7 +1173,7 @@ class SQLiteDialect(default.DefaultDialect): return [] table_data = row[0] - UNIQUE_PATTERN = 'CONSTRAINT (\w+) UNIQUE \(([^\)]+)\)' + UNIQUE_PATTERN = '(?:CONSTRAINT (\w+) )?UNIQUE \(([^\)]+)\)' return [ {'name': name, 'column_names': [col.strip(' "') for col in cols.split(',')]} -- cgit v1.2.1 From 85c04dd0bb9d0f140dde25e3901b172ebb431f7e Mon Sep 17 00:00:00 2001 From: Jon Nelson Date: Fri, 14 Nov 2014 19:53:28 -0600 Subject: - add test_get_unnamed_unique_constraints to SQLite reflection tests --- test/dialect/test_sqlite.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/dialect/test_sqlite.py b/test/dialect/test_sqlite.py index 22772d2fb..b4524dc27 100644 --- a/test/dialect/test_sqlite.py +++ b/test/dialect/test_sqlite.py @@ -603,6 +603,24 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): eq_(inspector.get_unique_constraints('bar'), [{'column_names': [u'b'], 'name': u'bar_b'}]) + def test_get_unnamed_unique_constraints(self): + meta = MetaData(testing.db) + t1 = Table('foo', meta, Column('f', Integer), + UniqueConstraint('f')) + t2 = Table('bar', meta, Column('b', Integer), + UniqueConstraint('b'), + prefixes=['TEMPORARY']) + meta.create_all() + from sqlalchemy.engine.reflection import Inspector + try: + inspector = Inspector(testing.db) + eq_(inspector.get_unique_constraints('foo'), + [{'column_names': [u'f'], 'name': u''}]) + eq_(inspector.get_unique_constraints('bar'), + [{'column_names': [u'b'], 'name': u''}]) + finally: + meta.drop_all() + class AttachedMemoryDBTest(fixtures.TestBase): __only_on__ = 'sqlite' -- cgit v1.2.1 From 5b146e1bab7b440038c356f388e3362a669399c1 Mon Sep 17 00:00:00 2001 From: Jon Nelson Date: Fri, 14 Nov 2014 20:05:58 -0600 Subject: - add tentative 'changelog' documentation on #3244 --- doc/build/changelog/changelog_09.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 419827959..f83afd2da 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -59,6 +59,15 @@ replaced, however if the mapping were already used for querying, the old relationship would still be referenced within some registries. + .. change:: + :tags: bug, sqlite + :versions: 1.0.0 + :tickets: 3244 + + Fixed issue where un-named UNIQUE constraints were not being + reflected in SQLite. Now un-named UNIQUE constraints are returned + with a name of u''. + .. change:: :tags: bug, sql :versions: 1.0.0 -- cgit v1.2.1 From 468db416dbf284f0e7dddde90ec9641dc89428c6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 13 Dec 2014 18:04:11 -0500 Subject: - rework sqlite FK and unique constraint system to combine both PRAGMA and regexp parsing of SQL in order to form a complete picture of constraints + their names. fixes #3244 fixes #3261 - factor various PRAGMA work to be centralized into one call --- doc/build/changelog/changelog_09.rst | 9 - doc/build/changelog/changelog_10.rst | 9 + doc/build/changelog/migration_10.rst | 19 ++ lib/sqlalchemy/dialects/sqlite/base.py | 299 +++++++++++++++--------- test/dialect/test_sqlite.py | 414 +++++++++++++++++++++++---------- 5 files changed, 506 insertions(+), 244 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index f83afd2da..419827959 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -59,15 +59,6 @@ replaced, however if the mapping were already used for querying, the old relationship would still be referenced within some registries. - .. change:: - :tags: bug, sqlite - :versions: 1.0.0 - :tickets: 3244 - - Fixed issue where un-named UNIQUE constraints were not being - reflected in SQLite. Now un-named UNIQUE constraints are returned - with a name of u''. - .. change:: :tags: bug, sql :versions: 1.0.0 diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index d6f36e97e..4da7b9456 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,15 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, sqlite + :tickets: 3244, 3261 + + UNIQUE and FOREIGN KEY constraints are now fully reflected on + SQLite both with and without names. Previously, foreign key + names were ignored and unnamed unique constraints were skipped. + Thanks to Jon Nelson for assistance with this. + .. change:: :tags: feature, examples diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index cd5d420e5..e1fb13662 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1680,6 +1680,25 @@ reflection from temp tables as well, which is :ticket:`3203`. :ticket:`3204` +SQLite named and unnamed UNIQUE and FOREIGN KEY constraints will inspect and reflect +------------------------------------------------------------------------------------- + +UNIQUE and FOREIGN KEY constraints are now fully reflected on +SQLite both with and without names. Previously, foreign key +names were ignored and unnamed unique constraints were skipped. In particular +this will help with Alembic's new SQLite migration features. + +To achieve this, for both foreign keys and unique constraints, the result +of PRAGMA foreign_keys, index_list, and index_info is combined with regular +expression parsing of the CREATE TABLE statement overall to form a complete +picture of the names of constraints, as well as differentiating UNIQUE +constraints that were created as UNIQUE vs. unnamed INDEXes. + +:ticket:`3244` + +:ticket:`3261` + + .. _change_3220: Improved support for CTEs in Oracle diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index 30d8a6ea3..e79299527 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -913,22 +913,9 @@ class SQLiteDialect(default.DefaultDialect): return [row[0] for row in rs] def has_table(self, connection, table_name, schema=None): - quote = self.identifier_preparer.quote_identifier - if schema is not None: - pragma = "PRAGMA %s." % quote(schema) - else: - pragma = "PRAGMA " - qtable = quote(table_name) - statement = "%stable_info(%s)" % (pragma, qtable) - cursor = _pragma_cursor(connection.execute(statement)) - row = cursor.fetchone() - - # consume remaining rows, to work around - # http://www.sqlite.org/cvstrac/tktview?tn=1884 - while not cursor.closed and cursor.fetchone() is not None: - pass - - return row is not None + info = self._get_table_pragma( + connection, "table_info", table_name, schema=schema) + return bool(info) @reflection.cache def get_view_names(self, connection, schema=None, **kw): @@ -970,18 +957,11 @@ class SQLiteDialect(default.DefaultDialect): @reflection.cache def get_columns(self, connection, table_name, schema=None, **kw): - quote = self.identifier_preparer.quote_identifier - if schema is not None: - pragma = "PRAGMA %s." % quote(schema) - else: - pragma = "PRAGMA " - qtable = quote(table_name) - statement = "%stable_info(%s)" % (pragma, qtable) - c = _pragma_cursor(connection.execute(statement)) + info = self._get_table_pragma( + connection, "table_info", table_name, schema=schema) - rows = c.fetchall() columns = [] - for row in rows: + for row in info: (name, type_, nullable, default, primary_key) = ( row[1], row[2].upper(), not row[3], row[4], row[5]) @@ -1068,92 +1048,192 @@ class SQLiteDialect(default.DefaultDialect): @reflection.cache def get_foreign_keys(self, connection, table_name, schema=None, **kw): - quote = self.identifier_preparer.quote_identifier - if schema is not None: - pragma = "PRAGMA %s." % quote(schema) - else: - pragma = "PRAGMA " - qtable = quote(table_name) - statement = "%sforeign_key_list(%s)" % (pragma, qtable) - c = _pragma_cursor(connection.execute(statement)) - fkeys = [] + # sqlite makes this *extremely difficult*. + # First, use the pragma to get the actual FKs. + pragma_fks = self._get_table_pragma( + connection, "foreign_key_list", + table_name, schema=schema + ) + fks = {} - while True: - row = c.fetchone() - if row is None: - break + + for row in pragma_fks: (numerical_id, rtbl, lcol, rcol) = ( row[0], row[2], row[3], row[4]) - self._parse_fk(fks, fkeys, numerical_id, rtbl, lcol, rcol) - return fkeys + if rcol is None: + rcol = lcol - def _parse_fk(self, fks, fkeys, numerical_id, rtbl, lcol, rcol): - # sqlite won't return rcol if the table was created with REFERENCES - # , no col - if rcol is None: - rcol = lcol + if self._broken_fk_pragma_quotes: + rtbl = re.sub(r'^[\"\[`\']|[\"\]`\']$', '', rtbl) - if self._broken_fk_pragma_quotes: - rtbl = re.sub(r'^[\"\[`\']|[\"\]`\']$', '', rtbl) + if numerical_id in fks: + fk = fks[numerical_id] + else: + fk = fks[numerical_id] = { + 'name': None, + 'constrained_columns': [], + 'referred_schema': None, + 'referred_table': rtbl, + 'referred_columns': [], + } + fks[numerical_id] = fk - try: - fk = fks[numerical_id] - except KeyError: - fk = { - 'name': None, - 'constrained_columns': [], - 'referred_schema': None, - 'referred_table': rtbl, - 'referred_columns': [], - } - fkeys.append(fk) - fks[numerical_id] = fk - - if lcol not in fk['constrained_columns']: fk['constrained_columns'].append(lcol) - if rcol not in fk['referred_columns']: fk['referred_columns'].append(rcol) - return fk + + def fk_sig(constrained_columns, referred_table, referred_columns): + return tuple(constrained_columns) + (referred_table,) + \ + tuple(referred_columns) + + # then, parse the actual SQL and attempt to find DDL that matches + # the names as well. SQLite saves the DDL in whatever format + # it was typed in as, so need to be liberal here. + + keys_by_signature = dict( + ( + fk_sig( + fk['constrained_columns'], + fk['referred_table'], fk['referred_columns']), + fk + ) for fk in fks.values() + ) + + table_data = self._get_table_sql(connection, table_name, schema=schema) + if table_data is None: + # system tables, etc. + return [] + + def parse_fks(): + FK_PATTERN = ( + '(?:CONSTRAINT (\w+) +)?' + 'FOREIGN KEY *\( *(.+?) *\) +' + 'REFERENCES +(?:(?:"(.+?)")|([a-z0-9_]+)) *\((.+?)\)' + ) + + for match in re.finditer(FK_PATTERN, table_data, re.I): + ( + constraint_name, constrained_columns, + referred_quoted_name, referred_name, + referred_columns) = match.group(1, 2, 3, 4, 5) + constrained_columns = list( + self._find_cols_in_sig(constrained_columns)) + if not referred_columns: + referred_columns = constrained_columns + else: + referred_columns = list( + self._find_cols_in_sig(referred_columns)) + referred_name = referred_quoted_name or referred_name + yield ( + constraint_name, constrained_columns, + referred_name, referred_columns) + fkeys = [] + + for ( + constraint_name, constrained_columns, + referred_name, referred_columns) in parse_fks(): + sig = fk_sig( + constrained_columns, referred_name, referred_columns) + if sig not in keys_by_signature: + util.warn( + "WARNING: SQL-parsed foreign key constraint " + "'%s' could not be located in PRAGMA " + "foreign_keys for table %s" % ( + sig, + table_name + )) + continue + key = keys_by_signature.pop(sig) + key['name'] = constraint_name + fkeys.append(key) + # assume the remainders are the unnamed, inline constraints, just + # use them as is as it's extremely difficult to parse inline + # constraints + fkeys.extend(keys_by_signature.values()) + return fkeys + + def _find_cols_in_sig(self, sig): + for match in re.finditer(r'(?:"(.+?)")|([a-z0-9_]+)', sig, re.I): + yield match.group(1) or match.group(2) + + @reflection.cache + def get_unique_constraints(self, connection, table_name, + schema=None, **kw): + + auto_index_by_sig = {} + for idx in self.get_indexes( + connection, table_name, schema=schema, + include_auto_indexes=True, **kw): + if not idx['name'].startswith("sqlite_autoindex"): + continue + sig = tuple(idx['column_names']) + auto_index_by_sig[sig] = idx + + table_data = self._get_table_sql( + connection, table_name, schema=schema, **kw) + if not table_data: + return [] + + unique_constraints = [] + + def parse_uqs(): + UNIQUE_PATTERN = '(?:CONSTRAINT (\w+) +)?UNIQUE *\((.+?)\)' + INLINE_UNIQUE_PATTERN = ( + '(?:(".+?")|([a-z0-9]+)) ' + '+[a-z0-9_ ]+? +UNIQUE') + + for match in re.finditer(UNIQUE_PATTERN, table_data, re.I): + name, cols = match.group(1, 2) + yield name, list(self._find_cols_in_sig(cols)) + + # we need to match inlines as well, as we seek to differentiate + # a UNIQUE constraint from a UNIQUE INDEX, even though these + # are kind of the same thing :) + for match in re.finditer(INLINE_UNIQUE_PATTERN, table_data, re.I): + cols = list( + self._find_cols_in_sig(match.group(1) or match.group(2))) + yield None, cols + + for name, cols in parse_uqs(): + sig = tuple(cols) + if sig in auto_index_by_sig: + auto_index_by_sig.pop(sig) + parsed_constraint = { + 'name': name, + 'column_names': cols + } + unique_constraints.append(parsed_constraint) + # NOTE: auto_index_by_sig might not be empty here, + # the PRIMARY KEY may have an entry. + return unique_constraints @reflection.cache def get_indexes(self, connection, table_name, schema=None, **kw): - quote = self.identifier_preparer.quote_identifier - if schema is not None: - pragma = "PRAGMA %s." % quote(schema) - else: - pragma = "PRAGMA " - include_auto_indexes = kw.pop('include_auto_indexes', False) - qtable = quote(table_name) - statement = "%sindex_list(%s)" % (pragma, qtable) - c = _pragma_cursor(connection.execute(statement)) + pragma_indexes = self._get_table_pragma( + connection, "index_list", table_name, schema=schema) indexes = [] - while True: - row = c.fetchone() - if row is None: - break + + include_auto_indexes = kw.pop('include_auto_indexes', False) + for row in pragma_indexes: # ignore implicit primary key index. # http://www.mail-archive.com/sqlite-users@sqlite.org/msg30517.html - elif (not include_auto_indexes and - row[1].startswith('sqlite_autoindex')): + if (not include_auto_indexes and + row[1].startswith('sqlite_autoindex')): continue indexes.append(dict(name=row[1], column_names=[], unique=row[2])) + # loop thru unique indexes to get the column names. for idx in indexes: - statement = "%sindex_info(%s)" % (pragma, quote(idx['name'])) - c = connection.execute(statement) - cols = idx['column_names'] - while True: - row = c.fetchone() - if row is None: - break - cols.append(row[2]) + pragma_index = self._get_table_pragma( + connection, "index_info", idx['name']) + + for row in pragma_index: + idx['column_names'].append(row[2]) return indexes @reflection.cache - def get_unique_constraints(self, connection, table_name, - schema=None, **kw): + def _get_table_sql(self, connection, table_name, schema=None, **kw): try: s = ("SELECT sql FROM " " (SELECT * FROM sqlite_master UNION ALL " @@ -1165,27 +1245,22 @@ class SQLiteDialect(default.DefaultDialect): s = ("SELECT sql FROM sqlite_master WHERE name = '%s' " "AND type = 'table'") % table_name rs = connection.execute(s) - row = rs.fetchone() - if row is None: - # sqlite won't return the schema for the sqlite_master or - # sqlite_temp_master tables from this query. These tables - # don't have any unique constraints anyway. - return [] - table_data = row[0] - - UNIQUE_PATTERN = '(?:CONSTRAINT (\w+) )?UNIQUE \(([^\)]+)\)' - return [ - {'name': name, - 'column_names': [col.strip(' "') for col in cols.split(',')]} - for name, cols in re.findall(UNIQUE_PATTERN, table_data) - ] + return rs.scalar() - -def _pragma_cursor(cursor): - """work around SQLite issue whereby cursor.description - is blank when PRAGMA returns no rows.""" - - if cursor.closed: - cursor.fetchone = lambda: None - cursor.fetchall = lambda: [] - return cursor + def _get_table_pragma(self, connection, pragma, table_name, schema=None): + quote = self.identifier_preparer.quote_identifier + if schema is not None: + statement = "PRAGMA %s." % quote(schema) + else: + statement = "PRAGMA " + qtable = quote(table_name) + statement = "%s%s(%s)" % (statement, pragma, qtable) + cursor = connection.execute(statement) + if not cursor.closed: + # work around SQLite issue whereby cursor.description + # is blank when PRAGMA returns no rows: + # http://www.sqlite.org/cvstrac/tktview?tn=1884 + result = cursor.fetchall() + else: + result = [] + return result diff --git a/test/dialect/test_sqlite.py b/test/dialect/test_sqlite.py index b4524dc27..44e4eda42 100644 --- a/test/dialect/test_sqlite.py +++ b/test/dialect/test_sqlite.py @@ -22,6 +22,7 @@ from sqlalchemy.testing import fixtures, AssertsCompiledSQL, \ from sqlalchemy import testing from sqlalchemy.schema import CreateTable from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.testing import mock class TestTypes(fixtures.TestBase, AssertsExecutionResults): @@ -500,30 +501,6 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): # assert j.onclause.compare(table1.c['"id"'] # == table2.c['"aid"']) - def test_legacy_quoted_identifiers_unit(self): - dialect = sqlite.dialect() - dialect._broken_fk_pragma_quotes = True - - for row in [ - (0, 'target', 'tid', 'id'), - (0, '"target"', 'tid', 'id'), - (0, '[target]', 'tid', 'id'), - (0, "'target'", 'tid', 'id'), - (0, '`target`', 'tid', 'id'), - ]: - fks = {} - fkeys = [] - dialect._parse_fk(fks, fkeys, *row) - eq_( - fkeys, - [{ - 'referred_table': 'target', - 'referred_columns': ['id'], - 'referred_schema': None, - 'name': None, - 'constrained_columns': ['tid'] - }]) - @testing.provide_metadata def test_description_encoding(self): # amazingly, pysqlite seems to still deliver cursor.description @@ -557,69 +534,6 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): e = create_engine('sqlite+pysqlite:///foo.db') assert e.pool.__class__ is pool.NullPool - @testing.provide_metadata - def test_dont_reflect_autoindex(self): - meta = self.metadata - Table('foo', meta, Column('bar', String, primary_key=True)) - meta.create_all() - inspector = Inspector(testing.db) - eq_(inspector.get_indexes('foo'), []) - eq_( - inspector.get_indexes('foo', include_auto_indexes=True), - [{ - 'unique': 1, - 'name': 'sqlite_autoindex_foo_1', - 'column_names': ['bar']}]) - - @testing.provide_metadata - def test_create_index_with_schema(self): - """Test creation of index with explicit schema""" - - meta = self.metadata - Table( - 'foo', meta, Column('bar', String, index=True), - schema='main') - meta.create_all() - inspector = Inspector(testing.db) - eq_( - inspector.get_indexes('foo', schema='main'), - [{'unique': 0, 'name': u'ix_main_foo_bar', - 'column_names': [u'bar']}]) - - @testing.provide_metadata - def test_get_unique_constraints(self): - meta = self.metadata - Table( - 'foo', meta, Column('f', Integer), - UniqueConstraint('f', name='foo_f')) - Table( - 'bar', meta, Column('b', Integer), - UniqueConstraint('b', name='bar_b'), - prefixes=['TEMPORARY']) - meta.create_all() - inspector = Inspector(testing.db) - eq_(inspector.get_unique_constraints('foo'), - [{'column_names': [u'f'], 'name': u'foo_f'}]) - eq_(inspector.get_unique_constraints('bar'), - [{'column_names': [u'b'], 'name': u'bar_b'}]) - - def test_get_unnamed_unique_constraints(self): - meta = MetaData(testing.db) - t1 = Table('foo', meta, Column('f', Integer), - UniqueConstraint('f')) - t2 = Table('bar', meta, Column('b', Integer), - UniqueConstraint('b'), - prefixes=['TEMPORARY']) - meta.create_all() - from sqlalchemy.engine.reflection import Inspector - try: - inspector = Inspector(testing.db) - eq_(inspector.get_unique_constraints('foo'), - [{'column_names': [u'f'], 'name': u''}]) - eq_(inspector.get_unique_constraints('bar'), - [{'column_names': [u'b'], 'name': u''}]) - finally: - meta.drop_all() class AttachedMemoryDBTest(fixtures.TestBase): @@ -1072,52 +986,306 @@ class ReflectHeadlessFKsTest(fixtures.TestBase): assert b.c.id.references(a.c.id) -class ReflectFKConstraintTest(fixtures.TestBase): +class ConstraintReflectionTest(fixtures.TestBase): __only_on__ = 'sqlite' - def setup(self): - testing.db.execute("CREATE TABLE a1 (id INTEGER PRIMARY KEY)") - testing.db.execute("CREATE TABLE a2 (id INTEGER PRIMARY KEY)") - testing.db.execute( - "CREATE TABLE b (id INTEGER PRIMARY KEY, " - "FOREIGN KEY(id) REFERENCES a1(id)," - "FOREIGN KEY(id) REFERENCES a2(id)" - ")") - testing.db.execute( - "CREATE TABLE c (id INTEGER, " - "CONSTRAINT bar PRIMARY KEY(id)," - "CONSTRAINT foo1 FOREIGN KEY(id) REFERENCES a1(id)," - "CONSTRAINT foo2 FOREIGN KEY(id) REFERENCES a2(id)" - ")") + @classmethod + def setup_class(cls): + with testing.db.begin() as conn: + + conn.execute("CREATE TABLE a1 (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE a2 (id INTEGER PRIMARY KEY)") + conn.execute( + "CREATE TABLE b (id INTEGER PRIMARY KEY, " + "FOREIGN KEY(id) REFERENCES a1(id)," + "FOREIGN KEY(id) REFERENCES a2(id)" + ")") + conn.execute( + "CREATE TABLE c (id INTEGER, " + "CONSTRAINT bar PRIMARY KEY(id)," + "CONSTRAINT foo1 FOREIGN KEY(id) REFERENCES a1(id)," + "CONSTRAINT foo2 FOREIGN KEY(id) REFERENCES a2(id)" + ")") + conn.execute( + # the lower casing + inline is intentional here + "CREATE TABLE d (id INTEGER, x INTEGER unique)") + conn.execute( + # the lower casing + inline is intentional here + 'CREATE TABLE d1 ' + '(id INTEGER, "some ( STUPID n,ame" INTEGER unique)') + conn.execute( + # the lower casing + inline is intentional here + 'CREATE TABLE d2 ( "some STUPID n,ame" INTEGER unique)') + conn.execute( + # the lower casing + inline is intentional here + 'CREATE TABLE d3 ( "some STUPID n,ame" INTEGER NULL unique)') + + conn.execute( + # lower casing + inline is intentional + "CREATE TABLE e (id INTEGER, x INTEGER references a2(id))") + conn.execute( + 'CREATE TABLE e1 (id INTEGER, "some ( STUPID n,ame" INTEGER ' + 'references a2 ("some ( STUPID n,ame"))') + conn.execute( + 'CREATE TABLE e2 (id INTEGER, ' + '"some ( STUPID n,ame" INTEGER NOT NULL ' + 'references a2 ("some ( STUPID n,ame"))') + + conn.execute( + "CREATE TABLE f (x INTEGER, CONSTRAINT foo_fx UNIQUE(x))" + ) + conn.execute( + "CREATE TEMPORARY TABLE g " + "(x INTEGER, CONSTRAINT foo_gx UNIQUE(x))" + ) + conn.execute( + # intentional broken casing + "CREATE TABLE h (x INTEGER, COnstraINT foo_hx unIQUE(x))" + ) + conn.execute( + "CREATE TABLE i (x INTEGER, y INTEGER, PRIMARY KEY(x, y))" + ) + conn.execute( + "CREATE TABLE j (id INTEGER, q INTEGER, p INTEGER, " + "PRIMARY KEY(id), FOreiGN KEY(q,p) REFERENCes i(x,y))" + ) + conn.execute( + "CREATE TABLE k (id INTEGER, q INTEGER, p INTEGER, " + "PRIMARY KEY(id), " + "conSTRAINT my_fk FOreiGN KEY ( q , p ) " + "REFERENCes i ( x , y ))" + ) - def teardown(self): - testing.db.execute("drop table c") - testing.db.execute("drop table b") - testing.db.execute("drop table a1") - testing.db.execute("drop table a2") + meta = MetaData() + Table( + 'l', meta, Column('bar', String, index=True), + schema='main') + + Table( + 'm', meta, + Column('id', Integer, primary_key=True), + Column('x', String(30)), + UniqueConstraint('x') + ) - def test_name_is_none(self): + Table( + 'n', meta, + Column('id', Integer, primary_key=True), + Column('x', String(30)), + UniqueConstraint('x'), + prefixes=['TEMPORARY'] + ) + + meta.create_all(conn) + + # will contain an "autoindex" + conn.execute("create table o (foo varchar(20) primary key)") + + @classmethod + def teardown_class(cls): + with testing.db.begin() as conn: + for name in [ + "m", "main.l", "k", "j", "i", "h", "g", "f", "e", "e1", + "d", "d1", "d2", "c", "b", "a1", "a2"]: + conn.execute("drop table %s" % name) + + def test_legacy_quoted_identifiers_unit(self): + dialect = sqlite.dialect() + dialect._broken_fk_pragma_quotes = True + + for row in [ + (0, None, 'target', 'tid', 'id', None), + (0, None, '"target"', 'tid', 'id', None), + (0, None, '[target]', 'tid', 'id', None), + (0, None, "'target'", 'tid', 'id', None), + (0, None, '`target`', 'tid', 'id', None), + ]: + def _get_table_pragma(*arg, **kw): + return [row] + + def _get_table_sql(*arg, **kw): + return "CREATE TABLE foo "\ + "(tid INTEGER, "\ + "FOREIGN KEY(tid) REFERENCES %s (id))" % row[2] + with mock.patch.object( + dialect, "_get_table_pragma", _get_table_pragma): + with mock.patch.object( + dialect, '_get_table_sql', _get_table_sql): + + fkeys = dialect.get_foreign_keys(None, 'foo') + eq_( + fkeys, + [{ + 'referred_table': 'target', + 'referred_columns': ['id'], + 'referred_schema': None, + 'name': None, + 'constrained_columns': ['tid'] + }]) + + def test_foreign_key_name_is_none(self): # and not "0" - meta = MetaData() - b = Table('b', meta, autoload=True, autoload_with=testing.db) + inspector = Inspector(testing.db) + fks = inspector.get_foreign_keys('b') eq_( - [con.name for con in b.constraints], - [None, None, None] + fks, + [ + {'referred_table': 'a1', 'referred_columns': ['id'], + 'referred_schema': None, 'name': None, + 'constrained_columns': ['id']}, + {'referred_table': 'a2', 'referred_columns': ['id'], + 'referred_schema': None, 'name': None, + 'constrained_columns': ['id']}, + ] ) - def test_name_not_none(self): - # we don't have names for PK constraints, - # it appears we get back None in the pragma for - # FKs also (also it doesn't even appear to be documented on - # sqlite's docs - # at http://www.sqlite.org/pragma.html#pragma_foreign_key_list - # how did we ever know that's the "name" field ??) + def test_foreign_key_name_is_not_none(self): + inspector = Inspector(testing.db) + fks = inspector.get_foreign_keys('c') + eq_( + fks, + [ + { + 'referred_table': 'a1', 'referred_columns': ['id'], + 'referred_schema': None, 'name': 'foo1', + 'constrained_columns': ['id']}, + { + 'referred_table': 'a2', 'referred_columns': ['id'], + 'referred_schema': None, 'name': 'foo2', + 'constrained_columns': ['id']}, + ] + ) - meta = MetaData() - c = Table('c', meta, autoload=True, autoload_with=testing.db) + def test_unnamed_inline_foreign_key(self): + inspector = Inspector(testing.db) + fks = inspector.get_foreign_keys('e') + eq_( + fks, + [{ + 'referred_table': 'a2', 'referred_columns': ['id'], + 'referred_schema': None, + 'name': None, 'constrained_columns': ['x'] + }] + ) + + def test_unnamed_inline_foreign_key_quoted(self): + inspector = Inspector(testing.db) + + inspector = Inspector(testing.db) + fks = inspector.get_foreign_keys('e1') + eq_( + fks, + [{ + 'referred_table': 'a2', + 'referred_columns': ['some ( STUPID n,ame'], + 'referred_schema': None, + 'name': None, 'constrained_columns': ['some ( STUPID n,ame'] + }] + ) + fks = inspector.get_foreign_keys('e2') + eq_( + fks, + [{ + 'referred_table': 'a2', + 'referred_columns': ['some ( STUPID n,ame'], + 'referred_schema': None, + 'name': None, 'constrained_columns': ['some ( STUPID n,ame'] + }] + ) + + def test_foreign_key_composite_broken_casing(self): + inspector = Inspector(testing.db) + fks = inspector.get_foreign_keys('j') + eq_( + fks, + [{ + 'referred_table': 'i', + 'referred_columns': ['x', 'y'], + 'referred_schema': None, 'name': None, + 'constrained_columns': ['q', 'p']}] + ) + fks = inspector.get_foreign_keys('k') + eq_( + fks, + [{'referred_table': 'i', 'referred_columns': ['x', 'y'], + 'referred_schema': None, 'name': 'my_fk', + 'constrained_columns': ['q', 'p']}] + ) + + def test_dont_reflect_autoindex(self): + inspector = Inspector(testing.db) + eq_(inspector.get_indexes('o'), []) + eq_( + inspector.get_indexes('o', include_auto_indexes=True), + [{ + 'unique': 1, + 'name': 'sqlite_autoindex_o_1', + 'column_names': ['foo']}]) + + def test_create_index_with_schema(self): + """Test creation of index with explicit schema""" + + inspector = Inspector(testing.db) + eq_( + inspector.get_indexes('l', schema='main'), + [{'unique': 0, 'name': u'ix_main_l_bar', + 'column_names': [u'bar']}]) + + def test_unique_constraint_named(self): + inspector = Inspector(testing.db) + eq_( + inspector.get_unique_constraints("f"), + [{'column_names': ['x'], 'name': 'foo_fx'}] + ) + + def test_unique_constraint_named_broken_casing(self): + inspector = Inspector(testing.db) + eq_( + inspector.get_unique_constraints("h"), + [{'column_names': ['x'], 'name': 'foo_hx'}] + ) + + def test_unique_constraint_named_broken_temp(self): + inspector = Inspector(testing.db) + eq_( + inspector.get_unique_constraints("g"), + [{'column_names': ['x'], 'name': 'foo_gx'}] + ) + + def test_unique_constraint_unnamed_inline(self): + inspector = Inspector(testing.db) + eq_( + inspector.get_unique_constraints("d"), + [{'column_names': ['x'], 'name': None}] + ) + + def test_unique_constraint_unnamed_inline_quoted(self): + inspector = Inspector(testing.db) + eq_( + inspector.get_unique_constraints("d1"), + [{'column_names': ['some ( STUPID n,ame'], 'name': None}] + ) + eq_( + inspector.get_unique_constraints("d2"), + [{'column_names': ['some STUPID n,ame'], 'name': None}] + ) + eq_( + inspector.get_unique_constraints("d3"), + [{'column_names': ['some STUPID n,ame'], 'name': None}] + ) + + def test_unique_constraint_unnamed_normal(self): + inspector = Inspector(testing.db) + eq_( + inspector.get_unique_constraints("m"), + [{'column_names': ['x'], 'name': None}] + ) + + def test_unique_constraint_unnamed_normal_temporary(self): + inspector = Inspector(testing.db) eq_( - set([con.name for con in c.constraints]), - set([None, None]) + inspector.get_unique_constraints("n"), + [{'column_names': ['x'], 'name': None}] ) -- cgit v1.2.1 From 7cd4362924dd0133a604d4a0c52f1566acbd31ff Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 14 Dec 2014 16:21:40 -0500 Subject: - automap isn't new anymore --- doc/build/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/index.rst b/doc/build/index.rst index b65755d43..205a5c12b 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -39,7 +39,7 @@ of Python objects, proceed first to the tutorial. :doc:`Declarative Extension ` | :doc:`Association Proxy ` | :doc:`Hybrid Attributes ` | - :doc:`Automap ` (**new**) | + :doc:`Automap ` | :doc:`Mutable Scalars ` | :doc:`Ordered List ` -- cgit v1.2.1 From d5f88ee9e51ceeaf4705d3b456b33b779cf25a5c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 14 Dec 2014 17:10:44 -0500 Subject: - rework the migration doc sections - small fixes in bulk docs --- doc/build/changelog/migration_10.rst | 1610 +++++++++++++++++----------------- doc/build/orm/session.rst | 8 +- 2 files changed, 812 insertions(+), 806 deletions(-) diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index e1fb13662..db0d270a1 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -8,7 +8,7 @@ What's New in SQLAlchemy 1.0? undergoing maintenance releases as of May, 2014, and SQLAlchemy version 1.0, as of yet unreleased. - Document last updated: December 8, 2014 + Document last updated: December 14, 2014 Introduction ============ @@ -17,13 +17,44 @@ This guide introduces what's new in SQLAlchemy version 1.0, and also documents changes which affect users migrating their applications from the 0.9 series of SQLAlchemy to 1.0. -Please carefully review -:ref:`behavioral_changes_orm_10` and :ref:`behavioral_changes_core_10` for -potentially backwards-incompatible changes. +Please carefully review the sections on behavioral changes for +potentially backwards-incompatible changes in behavior. -New Features -============ +New Features and Improvements - ORM +=================================== + +New Session Bulk INSERT/UPDATE API +---------------------------------- + +A new series of :class:`.Session` methods which provide hooks directly +into the unit of work's facility for emitting INSERT and UPDATE +statements has been created. When used correctly, this expert-oriented system +can allow ORM-mappings to be used to generate bulk insert and update +statements batched into executemany groups, allowing the statements +to proceed at speeds that rival direct use of the Core. + +.. seealso:: + + :ref:`bulk_operations` - introduction and full documentation + +:ticket:`3100` + +New Performance Example Suite +------------------------------ + +Inspired by the benchmarking done for the :ref:`bulk_operations` feature +as well as for the :ref:`faq_how_to_profile` section of the FAQ, a new +example section has been added which features several scripts designed +to illustrate the relative performance profile of various Core and ORM +techniques. The scripts are organized into use cases, and are packaged +under a single console interface such that any combination of demonstrations +can be run, dumping out timings, Python profile results and/or RunSnake profile +displays. + +.. seealso:: + + :ref:`examples_performance` .. _feature_3150: @@ -160,502 +191,169 @@ the polymorphic union of the base. :ticket:`3150` :ticket:`2670` :ticket:`3149` :ticket:`2952` :ticket:`3050` -.. _feature_3034: - -Select/Query LIMIT / OFFSET may be specified as an arbitrary SQL expression ----------------------------------------------------------------------------- - -The :meth:`.Select.limit` and :meth:`.Select.offset` methods now accept -any SQL expression, in addition to integer values, as arguments. The ORM -:class:`.Query` object also passes through any expression to the underlying -:class:`.Select` object. Typically -this is used to allow a bound parameter to be passed, which can be substituted -with a value later:: +ORM full object fetches 25% faster +---------------------------------- - sel = select([table]).limit(bindparam('mylimit')).offset(bindparam('myoffset')) +The mechanics of the ``loading.py`` module as well as the identity map +have undergone several passes of inlining, refactoring, and pruning, so +that a raw load of rows now populates ORM-based objects around 25% faster. +Assuming a 1M row table, a script like the following illustrates the type +of load that's improved the most:: -Dialects which don't support non-integer LIMIT or OFFSET expressions may continue -to not support this behavior; third party dialects may also need modification -in order to take advantage of the new behavior. A dialect which currently -uses the ``._limit`` or ``._offset`` attributes will continue to function -for those cases where the limit/offset was specified as a simple integer value. -However, when a SQL expression is specified, these two attributes will -instead raise a :class:`.CompileError` on access. A third-party dialect which -wishes to support the new feature should now call upon the ``._limit_clause`` -and ``._offset_clause`` attributes to receive the full SQL expression, rather -than the integer value. + import time + from sqlalchemy import Integer, Column, create_engine, Table + from sqlalchemy.orm import Session + from sqlalchemy.ext.declarative import declarative_base -.. _change_2051: + Base = declarative_base() -.. _feature_insert_from_select_defaults: + class Foo(Base): + __table__ = Table( + 'foo', Base.metadata, + Column('id', Integer, primary_key=True), + Column('a', Integer(), nullable=False), + Column('b', Integer(), nullable=False), + Column('c', Integer(), nullable=False), + ) -INSERT FROM SELECT now includes Python and SQL-expression defaults -------------------------------------------------------------------- + engine = create_engine( + 'mysql+mysqldb://scott:tiger@localhost/test', echo=True) -:meth:`.Insert.from_select` now includes Python and SQL-expression defaults if -otherwise unspecified; the limitation where non-server column defaults -aren't included in an INSERT FROM SELECT is now lifted and these -expressions are rendered as constants into the SELECT statement:: + sess = Session(engine) - from sqlalchemy import Table, Column, MetaData, Integer, select, func + now = time.time() - m = MetaData() + # avoid using all() so that we don't have the overhead of building + # a large list of full objects in memory + for obj in sess.query(Foo).yield_per(100).limit(1000000): + pass - t = Table( - 't', m, - Column('x', Integer), - Column('y', Integer, default=func.somefunction())) + print("Total time: %d" % (time.time() - now)) - stmt = select([t.c.x]) - print t.insert().from_select(['x'], stmt) +Local MacBookPro results bench from 19 seconds for 0.9 down to 14 seconds for +1.0. The :meth:`.Query.yield_per` call is always a good idea when batching +huge numbers of rows, as it prevents the Python interpreter from having +to allocate a huge amount of memory for all objects and their instrumentation +at once. Without the :meth:`.Query.yield_per`, the above script on the +MacBookPro is 31 seconds on 0.9 and 26 seconds on 1.0, the extra time spent +setting up very large memory buffers. -Will render:: +.. _feature_3176: - INSERT INTO t (x, y) SELECT t.x, somefunction() AS somefunction_1 - FROM t +New KeyedTuple implementation dramatically faster +------------------------------------------------- -The feature can be disabled using -:paramref:`.Insert.from_select.include_defaults`. +We took a look into the :class:`.KeyedTuple` implementation in the hopes +of improving queries like this:: -New Postgresql Table options ------------------------------ + rows = sess.query(Foo.a, Foo.b, Foo.c).all() -Added support for PG table options TABLESPACE, ON COMMIT, -WITH(OUT) OIDS, and INHERITS, when rendering DDL via -the :class:`.Table` construct. +The :class:`.KeyedTuple` class is used rather than Python's +``collections.namedtuple()``, because the latter has a very complex +type-creation routine that benchmarks much slower than :class:`.KeyedTuple`. +However, when fetching hundreds of thousands of rows, +``collections.namedtuple()`` quickly overtakes :class:`.KeyedTuple` which +becomes dramatically slower as instance invocation goes up. What to do? +A new type that hedges between the approaches of both. Benching +all three types for "size" (number of rows returned) and "num" +(number of distinct queries), the new "lightweight keyed tuple" either +outperforms both, or lags very slightly behind the faster object, based on +which scenario. In the "sweet spot", where we are both creating a good number +of new types as well as fetching a good number of rows, the lightweight +object totally smokes both namedtuple and KeyedTuple:: -.. seealso:: + ----------------- + size=10 num=10000 # few rows, lots of queries + namedtuple: 3.60302400589 # namedtuple falls over + keyedtuple: 0.255059957504 # KeyedTuple very fast + lw keyed tuple: 0.582715034485 # lw keyed trails right on KeyedTuple + ----------------- + size=100 num=1000 # <--- sweet spot + namedtuple: 0.365247011185 + keyedtuple: 0.24896979332 + lw keyed tuple: 0.0889317989349 # lw keyed blows both away! + ----------------- + size=10000 num=100 + namedtuple: 0.572599887848 + keyedtuple: 2.54251694679 + lw keyed tuple: 0.613876104355 + ----------------- + size=1000000 num=10 # few queries, lots of rows + namedtuple: 5.79669594765 # namedtuple very fast + keyedtuple: 28.856498003 # KeyedTuple falls over + lw keyed tuple: 6.74346804619 # lw keyed trails right on namedtuple - :ref:`postgresql_table_options` -:ticket:`2051` +:ticket:`3176` -New Session Bulk INSERT/UPDATE API ----------------------------------- +.. _feature_updatemany: -A new series of :class:`.Session` methods which provide hooks directly -into the unit of work's facility for emitting INSERT and UPDATE -statements has been created. When used correctly, this expert-oriented system -can allow ORM-mappings to be used to generate bulk insert and update -statements batched into executemany groups, allowing the statements -to proceed at speeds that rival direct use of the Core. +UPDATE statements are now batched with executemany() in a flush +---------------------------------------------------------------- -.. seealso:: +UPDATE statements can now be batched within an ORM flush +into more performant executemany() call, similarly to how INSERT +statements can be batched; this will be invoked within flush +based on the following criteria: - :ref:`bulk_operations` - introduction and full documentation +* two or more UPDATE statements in sequence involve the identical set of + columns to be modified. -:ticket:`3100` +* The statement has no embedded SQL expressions in the SET clause. -New Performance Example Suite ------------------------------- +* The mapping does not use a :paramref:`~.orm.mapper.version_id_col`, or + the backend dialect supports a "sane" rowcount for an executemany() + operation; most DBAPIs support this correctly now. -Inspired by the benchmarking done for the :ref:`bulk_operations` feature -as well as for the :ref:`faq_how_to_profile` section of the FAQ, a new -example section has been added which features several scripts designed -to illustrate the relative performance profile of various Core and ORM -techniques. The scripts are organized into use cases, and are packaged -under a single console interface such that any combination of demonstrations -can be run, dumping out timings, Python profile results and/or RunSnake profile -displays. +.. _feature_3178: -.. seealso:: - :ref:`examples_performance` +.. _bug_3035: +Session.get_bind() handles a wider variety of inheritance scenarios +------------------------------------------------------------------- -.. _feature_get_enums: +The :meth:`.Session.get_bind` method is invoked whenever a query or unit +of work flush process seeks to locate the database engine that corresponds +to a particular class. The method has been improved to handle a variety +of inheritance-oriented scenarios, including: -New get_enums() method with Postgresql Dialect ----------------------------------------------- +* Binding to a Mixin or Abstract Class:: -The :func:`.inspect` method returns a :class:`.PGInspector` object in the -case of Postgresql, which includes a new :meth:`.PGInspector.get_enums` -method that returns information on all available ``ENUM`` types:: + class MyClass(SomeMixin, Base): + __tablename__ = 'my_table' + # ... - from sqlalchemy import inspect, create_engine + session = Session(binds={SomeMixin: some_engine}) - engine = create_engine("postgresql+psycopg2://host/dbname") - insp = inspect(engine) - print(insp.get_enums()) -.. seealso:: +* Binding to inherited concrete subclasses individually based on table:: - :meth:`.PGInspector.get_enums` + class BaseClass(Base): + __tablename__ = 'base' -.. _feature_2891: + # ... -Postgresql Dialect reflects Materialized Views, Foreign Tables --------------------------------------------------------------- + class ConcreteSubClass(BaseClass): + __tablename__ = 'concrete' -Changes are as follows: + # ... -* the :class:`Table` construct with ``autoload=True`` will now match a name - that exists in the database as a materialized view or foriegn table. + __mapper_args__ = {'concrete': True} -* :meth:`.Inspector.get_view_names` will return plain and materialized view - names. -* :meth:`.Inspector.get_table_names` does **not** change for Postgresql, it - continues to return only the names of plain tables. + session = Session(binds={ + base_table: some_engine, + concrete_table: some_other_engine + }) -* A new method :meth:`.PGInspector.get_foreign_table_names` is added which - will return the names of tables that are specifically marked as "foreign" - in the Postgresql schema tables. -The change to reflection involves adding ``'m'`` and ``'f'`` to the list -of qualifiers we use when querying ``pg_class.relkind``, but this change -is new in 1.0.0 to avoid any backwards-incompatible surprises for those -running 0.9 in production. +:ticket:`3035` -:ticket:`2891` +.. _feature_2963: -.. _change_3264: - -Postgresql ``has_table()`` now works for temporary tables ---------------------------------------------------------- - -This is a simple fix such that "has table" for temporary tables now works, -so that code like the following may proceed:: - - from sqlalchemy import * - - metadata = MetaData() - user_tmp = Table( - "user_tmp", metadata, - Column("id", INT, primary_key=True), - Column('name', VARCHAR(50)), - prefixes=['TEMPORARY'] - ) - - e = create_engine("postgresql://scott:tiger@localhost/test", echo='debug') - with e.begin() as conn: - user_tmp.create(conn, checkfirst=True) - - # checkfirst will succeed - user_tmp.create(conn, checkfirst=True) - -The very unlikely case that this behavior will cause a non-failing application -to behave differently, is because Postgresql allows a non-temporary table -to silently overwrite a temporary table. So code like the following will -now act completely differently, no longer creating the real table following -the temporary table:: - - from sqlalchemy import * - - metadata = MetaData() - user_tmp = Table( - "user_tmp", metadata, - Column("id", INT, primary_key=True), - Column('name', VARCHAR(50)), - prefixes=['TEMPORARY'] - ) - - e = create_engine("postgresql://scott:tiger@localhost/test", echo='debug') - with e.begin() as conn: - user_tmp.create(conn, checkfirst=True) - - m2 = MetaData() - user = Table( - "user_tmp", m2, - Column("id", INT, primary_key=True), - Column('name', VARCHAR(50)), - ) - - # in 0.9, *will create* the new table, overwriting the old one. - # in 1.0, *will not create* the new table - user.create(conn, checkfirst=True) - -:ticket:`3264` - -.. _feature_gh134: - -Postgresql FILTER keyword -------------------------- - -The SQL standard FILTER keyword for aggregate functions is now supported -by Postgresql as of 9.4. SQLAlchemy allows this using -:meth:`.FunctionElement.filter`:: - - func.count(1).filter(True) - -.. seealso:: - - :meth:`.FunctionElement.filter` - - :class:`.FunctionFilter` - -.. _feature_3184: - -UniqueConstraint is now part of the Table reflection process ------------------------------------------------------------- - -A :class:`.Table` object populated using ``autoload=True`` will now -include :class:`.UniqueConstraint` constructs as well as -:class:`.Index` constructs. This logic has a few caveats for -Postgresql and Mysql: - -Postgresql -^^^^^^^^^^ - -Postgresql has the behavior such that when a UNIQUE constraint is -created, it implicitly creates a UNIQUE INDEX corresponding to that -constraint as well. The :meth:`.Inspector.get_indexes` and the -:meth:`.Inspector.get_unique_constraints` methods will continue to -**both** return these entries distinctly, where -:meth:`.Inspector.get_indexes` now features a token -``duplicates_constraint`` within the index entry indicating the -corresponding constraint when detected. However, when performing -full table reflection using ``Table(..., autoload=True)``, the -:class:`.Index` construct is detected as being linked to the -:class:`.UniqueConstraint`, and is **not** present within the -:attr:`.Table.indexes` collection; only the :class:`.UniqueConstraint` -will be present in the :attr:`.Table.constraints` collection. This -deduplication logic works by joining to the ``pg_constraint`` table -when querying ``pg_index`` to see if the two constructs are linked. - -MySQL -^^^^^ - -MySQL does not have separate concepts for a UNIQUE INDEX and a UNIQUE -constraint. While it supports both syntaxes when creating tables and indexes, -it does not store them any differently. The -:meth:`.Inspector.get_indexes` -and the :meth:`.Inspector.get_unique_constraints` methods will continue to -**both** return an entry for a UNIQUE index in MySQL, -where :meth:`.Inspector.get_unique_constraints` features a new token -``duplicates_index`` within the constraint entry indicating that this is a -dupe entry corresponding to that index. However, when performing -full table reflection using ``Table(..., autoload=True)``, -the :class:`.UniqueConstraint` construct is -**not** part of the fully reflected :class:`.Table` construct under any -circumstances; this construct is always represented by a :class:`.Index` -with the ``unique=True`` setting present in the :attr:`.Table.indexes` -collection. - -.. seealso:: - - :ref:`postgresql_index_reflection` - - :ref:`mysql_unique_constraints` - -:ticket:`3184` - - -Behavioral Improvements -======================= - -.. _feature_updatemany: - -UPDATE statements are now batched with executemany() in a flush ----------------------------------------------------------------- - -UPDATE statements can now be batched within an ORM flush -into more performant executemany() call, similarly to how INSERT -statements can be batched; this will be invoked within flush -based on the following criteria: - -* two or more UPDATE statements in sequence involve the identical set of - columns to be modified. - -* The statement has no embedded SQL expressions in the SET clause. - -* The mapping does not use a :paramref:`~.orm.mapper.version_id_col`, or - the backend dialect supports a "sane" rowcount for an executemany() - operation; most DBAPIs support this correctly now. - -ORM full object fetches 25% faster ----------------------------------- - -The mechanics of the ``loading.py`` module as well as the identity map -have undergone several passes of inlining, refactoring, and pruning, so -that a raw load of rows now populates ORM-based objects around 25% faster. -Assuming a 1M row table, a script like the following illustrates the type -of load that's improved the most:: - - import time - from sqlalchemy import Integer, Column, create_engine, Table - from sqlalchemy.orm import Session - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class Foo(Base): - __table__ = Table( - 'foo', Base.metadata, - Column('id', Integer, primary_key=True), - Column('a', Integer(), nullable=False), - Column('b', Integer(), nullable=False), - Column('c', Integer(), nullable=False), - ) - - engine = create_engine( - 'mysql+mysqldb://scott:tiger@localhost/test', echo=True) - - sess = Session(engine) - - now = time.time() - - # avoid using all() so that we don't have the overhead of building - # a large list of full objects in memory - for obj in sess.query(Foo).yield_per(100).limit(1000000): - pass - - print("Total time: %d" % (time.time() - now)) - -Local MacBookPro results bench from 19 seconds for 0.9 down to 14 seconds for -1.0. The :meth:`.Query.yield_per` call is always a good idea when batching -huge numbers of rows, as it prevents the Python interpreter from having -to allocate a huge amount of memory for all objects and their instrumentation -at once. Without the :meth:`.Query.yield_per`, the above script on the -MacBookPro is 31 seconds on 0.9 and 26 seconds on 1.0, the extra time spent -setting up very large memory buffers. - - -.. _feature_3176: - -New KeyedTuple implementation dramatically faster -------------------------------------------------- - -We took a look into the :class:`.KeyedTuple` implementation in the hopes -of improving queries like this:: - - rows = sess.query(Foo.a, Foo.b, Foo.c).all() - -The :class:`.KeyedTuple` class is used rather than Python's -``collections.namedtuple()``, because the latter has a very complex -type-creation routine that benchmarks much slower than :class:`.KeyedTuple`. -However, when fetching hundreds of thousands of rows, -``collections.namedtuple()`` quickly overtakes :class:`.KeyedTuple` which -becomes dramatically slower as instance invocation goes up. What to do? -A new type that hedges between the approaches of both. Benching -all three types for "size" (number of rows returned) and "num" -(number of distinct queries), the new "lightweight keyed tuple" either -outperforms both, or lags very slightly behind the faster object, based on -which scenario. In the "sweet spot", where we are both creating a good number -of new types as well as fetching a good number of rows, the lightweight -object totally smokes both namedtuple and KeyedTuple:: - - ----------------- - size=10 num=10000 # few rows, lots of queries - namedtuple: 3.60302400589 # namedtuple falls over - keyedtuple: 0.255059957504 # KeyedTuple very fast - lw keyed tuple: 0.582715034485 # lw keyed trails right on KeyedTuple - ----------------- - size=100 num=1000 # <--- sweet spot - namedtuple: 0.365247011185 - keyedtuple: 0.24896979332 - lw keyed tuple: 0.0889317989349 # lw keyed blows both away! - ----------------- - size=10000 num=100 - namedtuple: 0.572599887848 - keyedtuple: 2.54251694679 - lw keyed tuple: 0.613876104355 - ----------------- - size=1000000 num=10 # few queries, lots of rows - namedtuple: 5.79669594765 # namedtuple very fast - keyedtuple: 28.856498003 # KeyedTuple falls over - lw keyed tuple: 6.74346804619 # lw keyed trails right on namedtuple - - -:ticket:`3176` - -.. _bug_3035: - -Session.get_bind() handles a wider variety of inheritance scenarios -------------------------------------------------------------------- - -The :meth:`.Session.get_bind` method is invoked whenever a query or unit -of work flush process seeks to locate the database engine that corresponds -to a particular class. The method has been improved to handle a variety -of inheritance-oriented scenarios, including: - -* Binding to a Mixin or Abstract Class:: - - class MyClass(SomeMixin, Base): - __tablename__ = 'my_table' - # ... - - session = Session(binds={SomeMixin: some_engine}) - - -* Binding to inherited concrete subclasses individually based on table:: - - class BaseClass(Base): - __tablename__ = 'base' - - # ... - - class ConcreteSubClass(BaseClass): - __tablename__ = 'concrete' - - # ... - - __mapper_args__ = {'concrete': True} - - - session = Session(binds={ - base_table: some_engine, - concrete_table: some_other_engine - }) - - -:ticket:`3035` - - -.. _feature_3178: - -New systems to safely emit parameterized warnings -------------------------------------------------- - -For a long time, there has been a restriction that warning messages could not -refer to data elements, such that a particular function might emit an -infinite number of unique warnings. The key place this occurs is in the -``Unicode type received non-unicode bind param value`` warning. Placing -the data value in this message would mean that the Python ``__warningregistry__`` -for that module, or in some cases the Python-global ``warnings.onceregistry``, -would grow unbounded, as in most warning scenarios, one of these two collections -is populated with every distinct warning message. - -The change here is that by using a special ``string`` type that purposely -changes how the string is hashed, we can control that a large number of -parameterized messages are hashed only on a small set of possible hash -values, such that a warning such as ``Unicode type received non-unicode -bind param value`` can be tailored to be emitted only a specific number -of times; beyond that, the Python warnings registry will begin recording -them as duplicates. - -To illustrate, the following test script will show only ten warnings being -emitted for ten of the parameter sets, out of a total of 1000:: - - from sqlalchemy import create_engine, Unicode, select, cast - import random - import warnings - - e = create_engine("sqlite://") - - # Use the "once" filter (which is also the default for Python - # warnings). Exactly ten of these warnings will - # be emitted; beyond that, the Python warnings registry will accumulate - # new values as dupes of one of the ten existing. - warnings.filterwarnings("once") - - for i in range(1000): - e.execute(select([cast( - ('foo_%d' % random.randint(0, 1000000)).encode('ascii'), Unicode)])) - -The format of the warning here is:: - - /path/lib/sqlalchemy/sql/sqltypes.py:186: SAWarning: Unicode type received - non-unicode bind param value 'foo_4852'. (this warning may be - suppressed after 10 occurrences) - - -:ticket:`3178` - -.. _feature_2963: - -.info dictionary improvements ------------------------------ +.info dictionary improvements +----------------------------- The :attr:`.InspectionAttr.info` collection is now available on every kind of object that one would retrieve from the :attr:`.Mapper.all_orm_descriptors` @@ -683,128 +381,6 @@ as remaining ORM constructs such as :func:`.orm.synonym`. :ticket:`2963` -.. _migration_3177: - -Change to single-table-inheritance criteria when using from_self(), count() ---------------------------------------------------------------------------- - -Given a single-table inheritance mapping, such as:: - - class Widget(Base): - __table__ = 'widget_table' - - class FooWidget(Widget): - pass - -Using :meth:`.Query.from_self` or :meth:`.Query.count` against a subclass -would produce a subquery, but then add the "WHERE" criteria for subtypes -to the outside:: - - sess.query(FooWidget).from_self().all() - -rendering:: - - SELECT - anon_1.widgets_id AS anon_1_widgets_id, - anon_1.widgets_type AS anon_1_widgets_type - FROM (SELECT widgets.id AS widgets_id, widgets.type AS widgets_type, - FROM widgets) AS anon_1 - WHERE anon_1.widgets_type IN (?) - -The issue with this is that if the inner query does not specify all -columns, then we can't add the WHERE clause on the outside (it actually tries, -and produces a bad query). This decision -apparently goes way back to 0.6.5 with the note "may need to make more -adjustments to this". Well, those adjustments have arrived! So now the -above query will render:: - - SELECT - anon_1.widgets_id AS anon_1_widgets_id, - anon_1.widgets_type AS anon_1_widgets_type - FROM (SELECT widgets.id AS widgets_id, widgets.type AS widgets_type, - FROM widgets - WHERE widgets.type IN (?)) AS anon_1 - -So that queries that don't include "type" will still work!:: - - sess.query(FooWidget.id).count() - -Renders:: - - SELECT count(*) AS count_1 - FROM (SELECT widgets.id AS widgets_id - FROM widgets - WHERE widgets.type IN (?)) AS anon_1 - - -:ticket:`3177` - - -.. _migration_3222: - - -single-table-inheritance criteria added to all ON clauses unconditionally -------------------------------------------------------------------------- - -When joining to a single-table inheritance subclass target, the ORM always adds -the "single table criteria" when joining on a relationship. Given a -mapping as:: - - class Widget(Base): - __tablename__ = 'widget' - id = Column(Integer, primary_key=True) - type = Column(String) - related_id = Column(ForeignKey('related.id')) - related = relationship("Related", backref="widget") - __mapper_args__ = {'polymorphic_on': type} - - - class FooWidget(Widget): - __mapper_args__ = {'polymorphic_identity': 'foo'} - - - class Related(Base): - __tablename__ = 'related' - id = Column(Integer, primary_key=True) - -It's been the behavior for quite some time that a JOIN on the relationship -will render a "single inheritance" clause for the type:: - - s.query(Related).join(FooWidget, Related.widget).all() - -SQL output:: - - SELECT related.id AS related_id - FROM related JOIN widget ON related.id = widget.related_id AND widget.type IN (:type_1) - -Above, because we joined to a subclass ``FooWidget``, :meth:`.Query.join` -knew to add the ``AND widget.type IN ('foo')`` criteria to the ON clause. - -The change here is that the ``AND widget.type IN()`` criteria is now appended -to *any* ON clause, not just those generated from a relationship, -including one that is explicitly stated:: - - # ON clause will now render as - # related.id = widget.related_id AND widget.type IN (:type_1) - s.query(Related).join(FooWidget, FooWidget.related_id == Related.id).all() - -As well as the "implicit" join when no ON clause of any kind is stated:: - - # ON clause will now render as - # related.id = widget.related_id AND widget.type IN (:type_1) - s.query(Related).join(FooWidget).all() - -Previously, the ON clause for these would not include the single-inheritance -criteria. Applications that are already adding this criteria to work around -this will want to remove its explicit use, though it should continue to work -fine if the criteria happens to be rendered twice in the meantime. - -.. seealso:: - - :ref:`bug_3233` - -:ticket:`3222` - .. _bug_3188: ColumnProperty constructs work a lot better with aliases, order_by @@ -853,85 +429,204 @@ New output:: FROM b WHERE b.a_id = a_1.id) AS anon_2 FROM a, a AS a_1 ORDER BY anon_2 -There were also many scenarios where the "order by" logic would fail -to order by label, for example if the mapping were "polymorphic":: +There were also many scenarios where the "order by" logic would fail +to order by label, for example if the mapping were "polymorphic":: + + class A(Base): + __tablename__ = 'a' + + id = Column(Integer, primary_key=True) + type = Column(String) + + __mapper_args__ = {'polymorphic_on': type, 'with_polymorphic': '*'} + +The order_by would fail to use the label, as it would be anonymized due +to the polymorphic loading:: + + SELECT a.id AS a_id, a.type AS a_type, (SELECT max(b.id) AS max_1 + FROM b WHERE b.a_id = a.id) AS anon_1 + FROM a ORDER BY (SELECT max(b.id) AS max_2 + FROM b WHERE b.a_id = a.id) + +Now that the order by label tracks the anonymized label, this now works:: + + SELECT a.id AS a_id, a.type AS a_type, (SELECT max(b.id) AS max_1 + FROM b WHERE b.a_id = a.id) AS anon_1 + FROM a ORDER BY anon_1 + +Included in these fixes are a variety of heisenbugs that could corrupt +the state of an ``aliased()`` construct such that the labeling logic +would again fail; these have also been fixed. + +:ticket:`3148` :ticket:`3188` + +New Features and Improvements - Core +==================================== + +.. _feature_3034: + +Select/Query LIMIT / OFFSET may be specified as an arbitrary SQL expression +---------------------------------------------------------------------------- + +The :meth:`.Select.limit` and :meth:`.Select.offset` methods now accept +any SQL expression, in addition to integer values, as arguments. The ORM +:class:`.Query` object also passes through any expression to the underlying +:class:`.Select` object. Typically +this is used to allow a bound parameter to be passed, which can be substituted +with a value later:: + + sel = select([table]).limit(bindparam('mylimit')).offset(bindparam('myoffset')) + +Dialects which don't support non-integer LIMIT or OFFSET expressions may continue +to not support this behavior; third party dialects may also need modification +in order to take advantage of the new behavior. A dialect which currently +uses the ``._limit`` or ``._offset`` attributes will continue to function +for those cases where the limit/offset was specified as a simple integer value. +However, when a SQL expression is specified, these two attributes will +instead raise a :class:`.CompileError` on access. A third-party dialect which +wishes to support the new feature should now call upon the ``._limit_clause`` +and ``._offset_clause`` attributes to receive the full SQL expression, rather +than the integer value. + +.. _change_2051: + +.. _feature_insert_from_select_defaults: + +INSERT FROM SELECT now includes Python and SQL-expression defaults +------------------------------------------------------------------- + +:meth:`.Insert.from_select` now includes Python and SQL-expression defaults if +otherwise unspecified; the limitation where non-server column defaults +aren't included in an INSERT FROM SELECT is now lifted and these +expressions are rendered as constants into the SELECT statement:: + + from sqlalchemy import Table, Column, MetaData, Integer, select, func + + m = MetaData() + + t = Table( + 't', m, + Column('x', Integer), + Column('y', Integer, default=func.somefunction())) + + stmt = select([t.c.x]) + print t.insert().from_select(['x'], stmt) + +Will render:: + + INSERT INTO t (x, y) SELECT t.x, somefunction() AS somefunction_1 + FROM t - class A(Base): - __tablename__ = 'a' +The feature can be disabled using +:paramref:`.Insert.from_select.include_defaults`. - id = Column(Integer, primary_key=True) - type = Column(String) +.. _feature_3184: - __mapper_args__ = {'polymorphic_on': type, 'with_polymorphic': '*'} +UniqueConstraint is now part of the Table reflection process +------------------------------------------------------------ -The order_by would fail to use the label, as it would be anonymized due -to the polymorphic loading:: +A :class:`.Table` object populated using ``autoload=True`` will now +include :class:`.UniqueConstraint` constructs as well as +:class:`.Index` constructs. This logic has a few caveats for +Postgresql and Mysql: - SELECT a.id AS a_id, a.type AS a_type, (SELECT max(b.id) AS max_1 - FROM b WHERE b.a_id = a.id) AS anon_1 - FROM a ORDER BY (SELECT max(b.id) AS max_2 - FROM b WHERE b.a_id = a.id) +Postgresql +^^^^^^^^^^ -Now that the order by label tracks the anonymized label, this now works:: +Postgresql has the behavior such that when a UNIQUE constraint is +created, it implicitly creates a UNIQUE INDEX corresponding to that +constraint as well. The :meth:`.Inspector.get_indexes` and the +:meth:`.Inspector.get_unique_constraints` methods will continue to +**both** return these entries distinctly, where +:meth:`.Inspector.get_indexes` now features a token +``duplicates_constraint`` within the index entry indicating the +corresponding constraint when detected. However, when performing +full table reflection using ``Table(..., autoload=True)``, the +:class:`.Index` construct is detected as being linked to the +:class:`.UniqueConstraint`, and is **not** present within the +:attr:`.Table.indexes` collection; only the :class:`.UniqueConstraint` +will be present in the :attr:`.Table.constraints` collection. This +deduplication logic works by joining to the ``pg_constraint`` table +when querying ``pg_index`` to see if the two constructs are linked. - SELECT a.id AS a_id, a.type AS a_type, (SELECT max(b.id) AS max_1 - FROM b WHERE b.a_id = a.id) AS anon_1 - FROM a ORDER BY anon_1 +MySQL +^^^^^ -Included in these fixes are a variety of heisenbugs that could corrupt -the state of an ``aliased()`` construct such that the labeling logic -would again fail; these have also been fixed. +MySQL does not have separate concepts for a UNIQUE INDEX and a UNIQUE +constraint. While it supports both syntaxes when creating tables and indexes, +it does not store them any differently. The +:meth:`.Inspector.get_indexes` +and the :meth:`.Inspector.get_unique_constraints` methods will continue to +**both** return an entry for a UNIQUE index in MySQL, +where :meth:`.Inspector.get_unique_constraints` features a new token +``duplicates_index`` within the constraint entry indicating that this is a +dupe entry corresponding to that index. However, when performing +full table reflection using ``Table(..., autoload=True)``, +the :class:`.UniqueConstraint` construct is +**not** part of the fully reflected :class:`.Table` construct under any +circumstances; this construct is always represented by a :class:`.Index` +with the ``unique=True`` setting present in the :attr:`.Table.indexes` +collection. -:ticket:`3148` :ticket:`3188` +.. seealso:: -.. _bug_3170: + :ref:`postgresql_index_reflection` -null(), false() and true() constants are no longer singletons -------------------------------------------------------------- + :ref:`mysql_unique_constraints` -These three constants were changed to return a "singleton" value -in 0.9; unfortunately, that would lead to a query like the following -to not render as expected:: +:ticket:`3184` - select([null(), null()]) -rendering only ``SELECT NULL AS anon_1``, because the two :func:`.null` -constructs would come out as the same ``NULL`` object, and -SQLAlchemy's Core model is based on object identity in order to -determine lexical significance. The change in 0.9 had no -importance other than the desire to save on object overhead; in general, -an unnamed construct needs to stay lexically unique so that it gets -labeled uniquely. +New systems to safely emit parameterized warnings +------------------------------------------------- -:ticket:`3170` +For a long time, there has been a restriction that warning messages could not +refer to data elements, such that a particular function might emit an +infinite number of unique warnings. The key place this occurs is in the +``Unicode type received non-unicode bind param value`` warning. Placing +the data value in this message would mean that the Python ``__warningregistry__`` +for that module, or in some cases the Python-global ``warnings.onceregistry``, +would grow unbounded, as in most warning scenarios, one of these two collections +is populated with every distinct warning message. -.. _change_3266: +The change here is that by using a special ``string`` type that purposely +changes how the string is hashed, we can control that a large number of +parameterized messages are hashed only on a small set of possible hash +values, such that a warning such as ``Unicode type received non-unicode +bind param value`` can be tailored to be emitted only a specific number +of times; beyond that, the Python warnings registry will begin recording +them as duplicates. -DBAPI exception wrapping and handle_error() event improvements --------------------------------------------------------------- +To illustrate, the following test script will show only ten warnings being +emitted for ten of the parameter sets, out of a total of 1000:: -SQLAlchemy's wrapping of DBAPI exceptions was not taking place in the -case where a :class:`.Connection` object was invalidated, and then tried -to reconnect and encountered an error; this has been resolved. + from sqlalchemy import create_engine, Unicode, select, cast + import random + import warnings -Additionally, the recently added :meth:`.ConnectionEvents.handle_error` -event is now invoked for errors that occur upon initial connect, upon -reconnect, and when :func:`.create_engine` is used given a custom connection -function via :paramref:`.create_engine.creator`. + e = create_engine("sqlite://") -The :class:`.ExceptionContext` object has a new datamember -:attr:`.ExceptionContext.engine` that will always refer to the :class:`.Engine` -in use, in those cases when the :class:`.Connection` object is not available -(e.g. on initial connect). + # Use the "once" filter (which is also the default for Python + # warnings). Exactly ten of these warnings will + # be emitted; beyond that, the Python warnings registry will accumulate + # new values as dupes of one of the ten existing. + warnings.filterwarnings("once") + + for i in range(1000): + e.execute(select([cast( + ('foo_%d' % random.randint(0, 1000000)).encode('ascii'), Unicode)])) +The format of the warning here is:: -:ticket:`3266` + /path/lib/sqlalchemy/sql/sqltypes.py:186: SAWarning: Unicode type received + non-unicode bind param value 'foo_4852'. (this warning may be + suppressed after 10 occurrences) -.. _behavioral_changes_orm_10: +:ticket:`3178` -Behavioral Changes - ORM -======================== +Key Behavioral Changes - ORM +============================ .. _bug_3228: @@ -1302,40 +997,160 @@ OUTER joins despite the innerjoin directive:: FROM users LEFT OUTER JOIN orders ON LEFT OUTER JOIN items ON -As noted in the 0.9 notes, the only database backend that has difficulty -with right-nested joins is SQLite; SQLAlchemy as of 0.9 converts a right-nested -join into a subquery as a join target on SQLite. +As noted in the 0.9 notes, the only database backend that has difficulty +with right-nested joins is SQLite; SQLAlchemy as of 0.9 converts a right-nested +join into a subquery as a join target on SQLite. + +.. seealso:: + + :ref:`feature_2976` - description of the feature as introduced in 0.9.4. + +:ticket:`3008` + +query.update() with ``synchronize_session='evaluate'`` raises on multi-table update +----------------------------------------------------------------------------------- + +The "evaulator" for :meth:`.Query.update` won't work with multi-table +updates, and needs to be set to ``synchronize_session=False`` or +``synchronize_session='fetch'`` when multiple tables are present. +The new behavior is that an explicit exception is now raised, with a message +to change the synchronize setting. +This is upgraded from a warning emitted as of 0.9.7. + +:ticket:`3117` + +Resurrect Event has been Removed +-------------------------------- + +The "resurrect" ORM event has been removed entirely. This event ceased to +have any function since version 0.8 removed the older "mutable" system +from the unit of work. + + +.. _migration_3177: + +Change to single-table-inheritance criteria when using from_self(), count() +--------------------------------------------------------------------------- + +Given a single-table inheritance mapping, such as:: + + class Widget(Base): + __table__ = 'widget_table' + + class FooWidget(Widget): + pass + +Using :meth:`.Query.from_self` or :meth:`.Query.count` against a subclass +would produce a subquery, but then add the "WHERE" criteria for subtypes +to the outside:: + + sess.query(FooWidget).from_self().all() + +rendering:: + + SELECT + anon_1.widgets_id AS anon_1_widgets_id, + anon_1.widgets_type AS anon_1_widgets_type + FROM (SELECT widgets.id AS widgets_id, widgets.type AS widgets_type, + FROM widgets) AS anon_1 + WHERE anon_1.widgets_type IN (?) + +The issue with this is that if the inner query does not specify all +columns, then we can't add the WHERE clause on the outside (it actually tries, +and produces a bad query). This decision +apparently goes way back to 0.6.5 with the note "may need to make more +adjustments to this". Well, those adjustments have arrived! So now the +above query will render:: + + SELECT + anon_1.widgets_id AS anon_1_widgets_id, + anon_1.widgets_type AS anon_1_widgets_type + FROM (SELECT widgets.id AS widgets_id, widgets.type AS widgets_type, + FROM widgets + WHERE widgets.type IN (?)) AS anon_1 + +So that queries that don't include "type" will still work!:: + + sess.query(FooWidget.id).count() + +Renders:: + + SELECT count(*) AS count_1 + FROM (SELECT widgets.id AS widgets_id + FROM widgets + WHERE widgets.type IN (?)) AS anon_1 + + +:ticket:`3177` + + +.. _migration_3222: + + +single-table-inheritance criteria added to all ON clauses unconditionally +------------------------------------------------------------------------- + +When joining to a single-table inheritance subclass target, the ORM always adds +the "single table criteria" when joining on a relationship. Given a +mapping as:: + + class Widget(Base): + __tablename__ = 'widget' + id = Column(Integer, primary_key=True) + type = Column(String) + related_id = Column(ForeignKey('related.id')) + related = relationship("Related", backref="widget") + __mapper_args__ = {'polymorphic_on': type} + + + class FooWidget(Widget): + __mapper_args__ = {'polymorphic_identity': 'foo'} + + + class Related(Base): + __tablename__ = 'related' + id = Column(Integer, primary_key=True) + +It's been the behavior for quite some time that a JOIN on the relationship +will render a "single inheritance" clause for the type:: + + s.query(Related).join(FooWidget, Related.widget).all() + +SQL output:: -.. seealso:: + SELECT related.id AS related_id + FROM related JOIN widget ON related.id = widget.related_id AND widget.type IN (:type_1) - :ref:`feature_2976` - description of the feature as introduced in 0.9.4. +Above, because we joined to a subclass ``FooWidget``, :meth:`.Query.join` +knew to add the ``AND widget.type IN ('foo')`` criteria to the ON clause. -:ticket:`3008` +The change here is that the ``AND widget.type IN()`` criteria is now appended +to *any* ON clause, not just those generated from a relationship, +including one that is explicitly stated:: -query.update() with ``synchronize_session='evaluate'`` raises on multi-table update ------------------------------------------------------------------------------------ + # ON clause will now render as + # related.id = widget.related_id AND widget.type IN (:type_1) + s.query(Related).join(FooWidget, FooWidget.related_id == Related.id).all() -The "evaulator" for :meth:`.Query.update` won't work with multi-table -updates, and needs to be set to ``synchronize_session=False`` or -``synchronize_session='fetch'`` when multiple tables are present. -The new behavior is that an explicit exception is now raised, with a message -to change the synchronize setting. -This is upgraded from a warning emitted as of 0.9.7. +As well as the "implicit" join when no ON clause of any kind is stated:: -:ticket:`3117` + # ON clause will now render as + # related.id = widget.related_id AND widget.type IN (:type_1) + s.query(Related).join(FooWidget).all() -Resurrect Event has been Removed --------------------------------- +Previously, the ON clause for these would not include the single-inheritance +criteria. Applications that are already adding this criteria to work around +this will want to remove its explicit use, though it should continue to work +fine if the criteria happens to be rendered twice in the meantime. -The "resurrect" ORM event has been removed entirely. This event ceased to -have any function since version 0.8 removed the older "mutable" system -from the unit of work. +.. seealso:: + :ref:`bug_3233` -.. _behavioral_changes_core_10: +:ticket:`3222` -Behavioral Changes - Core -========================= +Key Behavioral Changes - Core +============================= .. _migration_2992: @@ -1438,128 +1253,329 @@ of this change we have enhanced its functionality. When we have a :func:`.select` or :class:`.Query` that refers to some column name or named label, we might want to GROUP BY and/or ORDER BY known columns or labels:: - stmt = select([ - user.c.name, - func.count(user.c.id).label("id_count") - ]).group_by("name").order_by("id_count") + stmt = select([ + user.c.name, + func.count(user.c.id).label("id_count") + ]).group_by("name").order_by("id_count") + +In the above statement we expect to see "ORDER BY id_count", as opposed to a +re-statement of the function. The string argument given is actively +matched to an entry in the columns clause during compilation, so the above +statement would produce as we expect, without warnings (though note that +the ``"name"`` expression has been resolved to ``users.name``!):: + + SELECT users.name, count(users.id) AS id_count + FROM users GROUP BY users.name ORDER BY id_count + +However, if we refer to a name that cannot be located, then we get +the warning again, as below:: + + stmt = select([ + user.c.name, + func.count(user.c.id).label("id_count") + ]).order_by("some_label") + +The output does what we say, but again it warns us:: + + SAWarning: Can't resolve label reference 'some_label'; converting to + text() (this warning may be suppressed after 10 occurrences) + + SELECT users.name, count(users.id) AS id_count + FROM users ORDER BY some_label + +The above behavior applies to all those places where we might want to refer +to a so-called "label reference"; ORDER BY and GROUP BY, but also within an +OVER clause as well as a DISTINCT ON clause that refers to columns (e.g. the +Postgresql syntax). + +We can still specify any arbitrary expression for ORDER BY or others using +:func:`.text`:: + + stmt = select([users]).order_by(text("some special expression")) + +The upshot of the whole change is that SQLAlchemy now would like us +to tell it when a string is sent that this string is explicitly +a :func:`.text` construct, or a column, table, etc., and if we use it as a +label name in an order by, group by, or other expression, SQLAlchemy expects +that the string resolves to something known, else it should again +be qualified with :func:`.text` or similar. + +:ticket:`2992` + +.. _change_3163: + +Event listeners can not be added or removed from within that event's runner +--------------------------------------------------------------------------- + +Removal of an event listener from inside that same event itself would +modify the elements of a list during iteration, which would cause +still-attached event listeners to silently fail to fire. To prevent +this while still maintaining performance, the lists have been replaced +with ``collections.deque()``, which does not allow any additions or +removals during iteration, and instead raises ``RuntimeError``. + +:ticket:`3163` + +.. _change_3169: + +The INSERT...FROM SELECT construct now implies ``inline=True`` +-------------------------------------------------------------- + +Using :meth:`.Insert.from_select` now implies ``inline=True`` +on :func:`.insert`. This helps to fix a bug where an +INSERT...FROM SELECT construct would inadvertently be compiled +as "implicit returning" on supporting backends, which would +cause breakage in the case of an INSERT that inserts zero rows +(as implicit returning expects a row), as well as arbitrary +return data in the case of an INSERT that inserts multiple +rows (e.g. only the first row of many). +A similar change is also applied to an INSERT..VALUES +with multiple parameter sets; implicit RETURNING will no longer emit +for this statement either. As both of these constructs deal +with varible numbers of rows, the +:attr:`.ResultProxy.inserted_primary_key` accessor does not +apply. Previously, there was a documentation note that one +may prefer ``inline=True`` with INSERT..FROM SELECT as some databases +don't support returning and therefore can't do "implicit" returning, +but there's no reason an INSERT...FROM SELECT needs implicit returning +in any case. Regular explicit :meth:`.Insert.returning` should +be used to return variable numbers of result rows if inserted +data is needed. + +:ticket:`3169` + +.. _change_3027: + +``autoload_with`` now implies ``autoload=True`` +----------------------------------------------- + +A :class:`.Table` can be set up for reflection by passing +:paramref:`.Table.autoload_with` alone:: + + my_table = Table('my_table', metadata, autoload_with=some_engine) + +:ticket:`3027` + +.. _change_3266: + +DBAPI exception wrapping and handle_error() event improvements +-------------------------------------------------------------- + +SQLAlchemy's wrapping of DBAPI exceptions was not taking place in the +case where a :class:`.Connection` object was invalidated, and then tried +to reconnect and encountered an error; this has been resolved. + +Additionally, the recently added :meth:`.ConnectionEvents.handle_error` +event is now invoked for errors that occur upon initial connect, upon +reconnect, and when :func:`.create_engine` is used given a custom connection +function via :paramref:`.create_engine.creator`. + +The :class:`.ExceptionContext` object has a new datamember +:attr:`.ExceptionContext.engine` that will always refer to the :class:`.Engine` +in use, in those cases when the :class:`.Connection` object is not available +(e.g. on initial connect). + + +:ticket:`3266` + +.. _change_3243: + +ForeignKeyConstraint.columns is now a ColumnCollection +------------------------------------------------------ + +:attr:`.ForeignKeyConstraint.columns` was previously a plain list +containing either strings or :class:`.Column` objects, depending on +how the :class:`.ForeignKeyConstraint` was constructed and whether it was +associated with a table. The collection is now a :class:`.ColumnCollection`, +and is only initialized after the :class:`.ForeignKeyConstraint` is +associated with a :class:`.Table`. A new accessor +:attr:`.ForeignKeyConstraint.column_keys` +is added to unconditionally return string keys for the local set of +columns regardless of how the object was constructed or its current +state. + + +.. _bug_3170: + +null(), false() and true() constants are no longer singletons +------------------------------------------------------------- + +These three constants were changed to return a "singleton" value +in 0.9; unfortunately, that would lead to a query like the following +to not render as expected:: + + select([null(), null()]) + +rendering only ``SELECT NULL AS anon_1``, because the two :func:`.null` +constructs would come out as the same ``NULL`` object, and +SQLAlchemy's Core model is based on object identity in order to +determine lexical significance. The change in 0.9 had no +importance other than the desire to save on object overhead; in general, +an unnamed construct needs to stay lexically unique so that it gets +labeled uniquely. + +:ticket:`3170` + +.. _change_3204: + +SQLite/Oracle have distinct methods for temporary table/view name reporting +--------------------------------------------------------------------------- + +The :meth:`.Inspector.get_table_names` and :meth:`.Inspector.get_view_names` +methods in the case of SQLite/Oracle would also return the names of temporary +tables and views, which is not provided by any other dialect (in the case +of MySQL at least it is not even possible). This logic has been moved +out to two new methods :meth:`.Inspector.get_temp_table_names` and +:meth:`.Inspector.get_temp_view_names`. + +Note that reflection of a specific named temporary table or temporary view, +either by ``Table('name', autoload=True)`` or via methods like +:meth:`.Inspector.get_columns` continues to function for most if not all +dialects. For SQLite specifically, there is a bug fix for UNIQUE constraint +reflection from temp tables as well, which is :ticket:`3203`. + +:ticket:`3204` + +Dialect Improvements and Changes - Postgresql +============================================= + +New Postgresql Table options +----------------------------- + +Added support for PG table options TABLESPACE, ON COMMIT, +WITH(OUT) OIDS, and INHERITS, when rendering DDL via +the :class:`.Table` construct. + +.. seealso:: + + :ref:`postgresql_table_options` + +:ticket:`2051` + +.. _feature_get_enums: + +New get_enums() method with Postgresql Dialect +---------------------------------------------- + +The :func:`.inspect` method returns a :class:`.PGInspector` object in the +case of Postgresql, which includes a new :meth:`.PGInspector.get_enums` +method that returns information on all available ``ENUM`` types:: + + from sqlalchemy import inspect, create_engine + + engine = create_engine("postgresql+psycopg2://host/dbname") + insp = inspect(engine) + print(insp.get_enums()) + +.. seealso:: + + :meth:`.PGInspector.get_enums` + +.. _feature_2891: -In the above statement we expect to see "ORDER BY id_count", as opposed to a -re-statement of the function. The string argument given is actively -matched to an entry in the columns clause during compilation, so the above -statement would produce as we expect, without warnings (though note that -the ``"name"`` expression has been resolved to ``users.name``!):: +Postgresql Dialect reflects Materialized Views, Foreign Tables +-------------------------------------------------------------- - SELECT users.name, count(users.id) AS id_count - FROM users GROUP BY users.name ORDER BY id_count +Changes are as follows: -However, if we refer to a name that cannot be located, then we get -the warning again, as below:: +* the :class:`Table` construct with ``autoload=True`` will now match a name + that exists in the database as a materialized view or foriegn table. - stmt = select([ - user.c.name, - func.count(user.c.id).label("id_count") - ]).order_by("some_label") +* :meth:`.Inspector.get_view_names` will return plain and materialized view + names. -The output does what we say, but again it warns us:: +* :meth:`.Inspector.get_table_names` does **not** change for Postgresql, it + continues to return only the names of plain tables. - SAWarning: Can't resolve label reference 'some_label'; converting to - text() (this warning may be suppressed after 10 occurrences) +* A new method :meth:`.PGInspector.get_foreign_table_names` is added which + will return the names of tables that are specifically marked as "foreign" + in the Postgresql schema tables. - SELECT users.name, count(users.id) AS id_count - FROM users ORDER BY some_label +The change to reflection involves adding ``'m'`` and ``'f'`` to the list +of qualifiers we use when querying ``pg_class.relkind``, but this change +is new in 1.0.0 to avoid any backwards-incompatible surprises for those +running 0.9 in production. -The above behavior applies to all those places where we might want to refer -to a so-called "label reference"; ORDER BY and GROUP BY, but also within an -OVER clause as well as a DISTINCT ON clause that refers to columns (e.g. the -Postgresql syntax). +:ticket:`2891` -We can still specify any arbitrary expression for ORDER BY or others using -:func:`.text`:: +.. _change_3264: - stmt = select([users]).order_by(text("some special expression")) +Postgresql ``has_table()`` now works for temporary tables +--------------------------------------------------------- -The upshot of the whole change is that SQLAlchemy now would like us -to tell it when a string is sent that this string is explicitly -a :func:`.text` construct, or a column, table, etc., and if we use it as a -label name in an order by, group by, or other expression, SQLAlchemy expects -that the string resolves to something known, else it should again -be qualified with :func:`.text` or similar. +This is a simple fix such that "has table" for temporary tables now works, +so that code like the following may proceed:: -:ticket:`2992` + from sqlalchemy import * -.. _change_3163: + metadata = MetaData() + user_tmp = Table( + "user_tmp", metadata, + Column("id", INT, primary_key=True), + Column('name', VARCHAR(50)), + prefixes=['TEMPORARY'] + ) -Event listeners can not be added or removed from within that event's runner ---------------------------------------------------------------------------- + e = create_engine("postgresql://scott:tiger@localhost/test", echo='debug') + with e.begin() as conn: + user_tmp.create(conn, checkfirst=True) -Removal of an event listener from inside that same event itself would -modify the elements of a list during iteration, which would cause -still-attached event listeners to silently fail to fire. To prevent -this while still maintaining performance, the lists have been replaced -with ``collections.deque()``, which does not allow any additions or -removals during iteration, and instead raises ``RuntimeError``. + # checkfirst will succeed + user_tmp.create(conn, checkfirst=True) -:ticket:`3163` +The very unlikely case that this behavior will cause a non-failing application +to behave differently, is because Postgresql allows a non-temporary table +to silently overwrite a temporary table. So code like the following will +now act completely differently, no longer creating the real table following +the temporary table:: -.. _change_3169: + from sqlalchemy import * -The INSERT...FROM SELECT construct now implies ``inline=True`` --------------------------------------------------------------- + metadata = MetaData() + user_tmp = Table( + "user_tmp", metadata, + Column("id", INT, primary_key=True), + Column('name', VARCHAR(50)), + prefixes=['TEMPORARY'] + ) -Using :meth:`.Insert.from_select` now implies ``inline=True`` -on :func:`.insert`. This helps to fix a bug where an -INSERT...FROM SELECT construct would inadvertently be compiled -as "implicit returning" on supporting backends, which would -cause breakage in the case of an INSERT that inserts zero rows -(as implicit returning expects a row), as well as arbitrary -return data in the case of an INSERT that inserts multiple -rows (e.g. only the first row of many). -A similar change is also applied to an INSERT..VALUES -with multiple parameter sets; implicit RETURNING will no longer emit -for this statement either. As both of these constructs deal -with varible numbers of rows, the -:attr:`.ResultProxy.inserted_primary_key` accessor does not -apply. Previously, there was a documentation note that one -may prefer ``inline=True`` with INSERT..FROM SELECT as some databases -don't support returning and therefore can't do "implicit" returning, -but there's no reason an INSERT...FROM SELECT needs implicit returning -in any case. Regular explicit :meth:`.Insert.returning` should -be used to return variable numbers of result rows if inserted -data is needed. + e = create_engine("postgresql://scott:tiger@localhost/test", echo='debug') + with e.begin() as conn: + user_tmp.create(conn, checkfirst=True) -:ticket:`3169` + m2 = MetaData() + user = Table( + "user_tmp", m2, + Column("id", INT, primary_key=True), + Column('name', VARCHAR(50)), + ) -.. _change_3027: + # in 0.9, *will create* the new table, overwriting the old one. + # in 1.0, *will not create* the new table + user.create(conn, checkfirst=True) -``autoload_with`` now implies ``autoload=True`` ------------------------------------------------ +:ticket:`3264` -A :class:`.Table` can be set up for reflection by passing -:paramref:`.Table.autoload_with` alone:: +.. _feature_gh134: - my_table = Table('my_table', metadata, autoload_with=some_engine) +Postgresql FILTER keyword +------------------------- -:ticket:`3027` +The SQL standard FILTER keyword for aggregate functions is now supported +by Postgresql as of 9.4. SQLAlchemy allows this using +:meth:`.FunctionElement.filter`:: -.. _change_3243: + func.count(1).filter(True) -ForeignKeyConstraint.columns is now a ColumnCollection ------------------------------------------------------- +.. seealso:: -:attr:`.ForeignKeyConstraint.columns` was previously a plain list -containing either strings or :class:`.Column` objects, depending on -how the :class:`.ForeignKeyConstraint` was constructed and whether it was -associated with a table. The collection is now a :class:`.ColumnCollection`, -and is only initialized after the :class:`.ForeignKeyConstraint` is -associated with a :class:`.Table`. A new accessor -:attr:`.ForeignKeyConstraint.column_keys` -is added to unconditionally return string keys for the local set of -columns regardless of how the object was constructed or its current -state. + :meth:`.FunctionElement.filter` + + :class:`.FunctionFilter` -Dialect Changes -=============== +Dialect Improvements and Changes - MySQL +============================================= MySQL internal "no such table" exceptions not passed to event handlers ---------------------------------------------------------------------- @@ -1634,6 +1650,48 @@ on MySQL:: :ticket:`3263` +.. _change_2984: + +Drizzle Dialect is now an External Dialect +------------------------------------------ + +The dialect for `Drizzle `_ is now an external +dialect, available at https://bitbucket.org/zzzeek/sqlalchemy-drizzle. +This dialect was added to SQLAlchemy right before SQLAlchemy was able to +accommodate third party dialects well; going forward, all databases that aren't +within the "ubiquitous use" category are third party dialects. +The dialect's implementation hasn't changed and is still based on the +MySQL + MySQLdb dialects within SQLAlchemy. The dialect is as of yet +unreleased and in "attic" status; however it passes the majority of tests +and is generally in decent working order, if someone wants to pick up +on polishing it. + +Dialect Improvements and Changes - SQLite +============================================= + +.. _change_2984: + +SQLite named and unnamed UNIQUE and FOREIGN KEY constraints will inspect and reflect +------------------------------------------------------------------------------------- + +UNIQUE and FOREIGN KEY constraints are now fully reflected on +SQLite both with and without names. Previously, foreign key +names were ignored and unnamed unique constraints were skipped. In particular +this will help with Alembic's new SQLite migration features. + +To achieve this, for both foreign keys and unique constraints, the result +of PRAGMA foreign_keys, index_list, and index_info is combined with regular +expression parsing of the CREATE TABLE statement overall to form a complete +picture of the names of constraints, as well as differentiating UNIQUE +constraints that were created as UNIQUE vs. unnamed INDEXes. + +:ticket:`3244` + +:ticket:`3261` + +Dialect Improvements and Changes - SQL Server +============================================= + .. _change_3182: PyODBC driver name is required with hostname-based SQL Server connections @@ -1660,44 +1718,8 @@ types has been changed for SQL Server 2012 and greater, with options to control the behavior completely, based on deprecation guidelines from Microsoft. See :ref:`mssql_large_type_deprecation` for details. -.. _change_3204: - -SQLite/Oracle have distinct methods for temporary table/view name reporting ---------------------------------------------------------------------------- - -The :meth:`.Inspector.get_table_names` and :meth:`.Inspector.get_view_names` -methods in the case of SQLite/Oracle would also return the names of temporary -tables and views, which is not provided by any other dialect (in the case -of MySQL at least it is not even possible). This logic has been moved -out to two new methods :meth:`.Inspector.get_temp_table_names` and -:meth:`.Inspector.get_temp_view_names`. - -Note that reflection of a specific named temporary table or temporary view, -either by ``Table('name', autoload=True)`` or via methods like -:meth:`.Inspector.get_columns` continues to function for most if not all -dialects. For SQLite specifically, there is a bug fix for UNIQUE constraint -reflection from temp tables as well, which is :ticket:`3203`. - -:ticket:`3204` - -SQLite named and unnamed UNIQUE and FOREIGN KEY constraints will inspect and reflect -------------------------------------------------------------------------------------- - -UNIQUE and FOREIGN KEY constraints are now fully reflected on -SQLite both with and without names. Previously, foreign key -names were ignored and unnamed unique constraints were skipped. In particular -this will help with Alembic's new SQLite migration features. - -To achieve this, for both foreign keys and unique constraints, the result -of PRAGMA foreign_keys, index_list, and index_info is combined with regular -expression parsing of the CREATE TABLE statement overall to form a complete -picture of the names of constraints, as well as differentiating UNIQUE -constraints that were created as UNIQUE vs. unnamed INDEXes. - -:ticket:`3244` - -:ticket:`3261` - +Dialect Improvements and Changes - Oracle +============================================= .. _change_3220: @@ -1725,19 +1747,3 @@ Keywords such as COMPRESS, ON COMMIT, BITMAP: :ref:`oracle_table_options` :ref:`oracle_index_options` - -.. _change_2984: - -Drizzle Dialect is now an External Dialect ------------------------------------------- - -The dialect for `Drizzle `_ is now an external -dialect, available at https://bitbucket.org/zzzeek/sqlalchemy-drizzle. -This dialect was added to SQLAlchemy right before SQLAlchemy was able to -accommodate third party dialects well; going forward, all databases that aren't -within the "ubiquitous use" category are third party dialects. -The dialect's implementation hasn't changed and is still based on the -MySQL + MySQLdb dialects within SQLAlchemy. The dialect is as of yet -unreleased and in "attic" status; however it passes the majority of tests -and is generally in decent working order, if someone wants to pick up -on polishing it. diff --git a/doc/build/orm/session.rst b/doc/build/orm/session.rst index 08ef9303e..97506e210 100644 --- a/doc/build/orm/session.rst +++ b/doc/build/orm/session.rst @@ -1944,7 +1944,7 @@ transactions set the flag ``twophase=True`` on the session:: # before committing both transactions session.commit() -.. _session_sql_expressions: +.. _flush_embedded_sql_expressions: Embedding SQL Insert/Update Expressions into a Flush ===================================================== @@ -2567,7 +2567,7 @@ The bulk insert / update methods lose a significant amount of functionality versus traditional ORM use. The following is a listing of features that are **not available** when using these methods: -* persistence along :meth:`.relationship` linkages +* persistence along :func:`.relationship` linkages * sorting of rows within order of dependency; rows are inserted or updated directly in the order in which they are passed to the methods @@ -2577,12 +2577,12 @@ are **not available** when using these methods: * Functionality related to primary key mutation, ON UPDATE cascade -* SQL expression inserts / updates (e.g. :ref:`session_sql_expressions`) +* SQL expression inserts / updates (e.g. :ref:`flush_embedded_sql_expressions`) * ORM events such as :meth:`.MapperEvents.before_insert`, etc. The bulk session methods have no event support. -Features that **are available** include:: +Features that **are available** include: * INSERTs and UPDATEs of mapped objects -- cgit v1.2.1 From 00aaaa4bd4aa150ff9964bf2c00b1404d2e8a140 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 16 Dec 2014 17:02:48 -0500 Subject: - Added a version check to the MySQLdb dialect surrounding the check for 'utf8_bin' collation, as this fails on MySQL server < 5.0. fixes #3274 --- doc/build/changelog/changelog_09.rst | 8 ++++++++ lib/sqlalchemy/dialects/mysql/mysqldb.py | 13 +++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 419827959..b2c876141 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,14 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, mysql + :versions: 1.0.0 + :tickets: 3274 + + Added a version check to the MySQLdb dialect surrounding the + check for 'utf8_bin' collation, as this fails on MySQL server < 5.0. + .. change:: :tags: enhancement, orm :versions: 1.0.0 diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index 893c6a9e2..5bb67a24d 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -102,12 +102,13 @@ class MySQLDialect_mysqldb(MySQLDialect): # https://github.com/farcepest/MySQLdb1/commit/cd44524fef63bd3fcb71947392326e9742d520e8 # specific issue w/ the utf8_bin collation and unicode returns - has_utf8_bin = connection.scalar( - "show collation where %s = 'utf8' and %s = 'utf8_bin'" - % ( - self.identifier_preparer.quote("Charset"), - self.identifier_preparer.quote("Collation") - )) + has_utf8_bin = self.server_version_info > (5, ) and \ + connection.scalar( + "show collation where %s = 'utf8' and %s = 'utf8_bin'" + % ( + self.identifier_preparer.quote("Charset"), + self.identifier_preparer.quote("Collation") + )) if has_utf8_bin: additional_tests = [ sql.collate(sql.cast( -- cgit v1.2.1 From 9561321d0328df270c4ff0360dc7a035db627949 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 17 Dec 2014 17:24:23 -0500 Subject: - squash-merge the improve_toc branch, which moves all the Sphinx styling and extensions into an external library, and also reorganizes most large documentation pages into many small areas to reduce scrolling and better present the context into a more fine-grained hierarchy. --- doc/build/builder/__init__.py | 0 doc/build/builder/autodoc_mods.py | 102 - doc/build/builder/dialect_info.py | 175 -- doc/build/builder/mako.py | 65 - doc/build/builder/sqlformatter.py | 132 - doc/build/builder/util.py | 12 - doc/build/builder/viewsource.py | 209 -- doc/build/conf.py | 25 +- doc/build/contents.rst | 1 + doc/build/core/api_basics.rst | 12 + doc/build/core/custom_types.rst | 498 ++++ doc/build/core/engines_connections.rst | 11 + doc/build/core/expression_api.rst | 4 +- doc/build/core/index.rst | 16 +- doc/build/core/schema.rst | 4 +- doc/build/core/type_api.rst | 22 + doc/build/core/type_basics.rst | 229 ++ doc/build/core/types.rst | 748 +----- doc/build/faq.rst | 1504 ----------- doc/build/faq/connections.rst | 138 + doc/build/faq/index.rst | 19 + doc/build/faq/metadata_schema.rst | 102 + doc/build/faq/ormconfiguration.rst | 334 +++ doc/build/faq/performance.rst | 443 ++++ doc/build/faq/sessions.rst | 363 +++ doc/build/faq/sqlexpressions.rst | 140 + doc/build/index.rst | 38 +- doc/build/orm/backref.rst | 273 ++ doc/build/orm/basic_relationships.rst | 313 +++ doc/build/orm/cascades.rst | 372 +++ doc/build/orm/classical.rst | 68 + doc/build/orm/composites.rst | 160 ++ doc/build/orm/constructors.rst | 56 + doc/build/orm/contextual.rst | 260 ++ doc/build/orm/extending.rst | 12 + doc/build/orm/extensions/declarative.rst | 33 - doc/build/orm/extensions/declarative/api.rst | 114 + doc/build/orm/extensions/declarative/basic_use.rst | 133 + doc/build/orm/extensions/declarative/index.rst | 32 + .../orm/extensions/declarative/inheritance.rst | 318 +++ doc/build/orm/extensions/declarative/mixins.rst | 541 ++++ .../orm/extensions/declarative/relationships.rst | 138 + .../orm/extensions/declarative/table_config.rst | 143 ++ doc/build/orm/extensions/index.rst | 2 +- doc/build/orm/index.rst | 11 +- doc/build/orm/join_conditions.rst | 740 ++++++ doc/build/orm/loading.rst | 546 ---- doc/build/orm/loading_columns.rst | 195 ++ doc/build/orm/loading_objects.rst | 15 + doc/build/orm/loading_relationships.rst | 546 ++++ doc/build/orm/mapped_attributes.rst | 340 +++ doc/build/orm/mapped_sql_expr.rst | 208 ++ doc/build/orm/mapper_config.rst | 1667 +----------- doc/build/orm/mapping_api.rst | 22 + doc/build/orm/mapping_columns.rst | 222 ++ doc/build/orm/nonstandard_mappings.rst | 152 ++ doc/build/orm/persistence_techniques.rst | 301 +++ doc/build/orm/query.rst | 12 +- doc/build/orm/relationship_api.rst | 19 + doc/build/orm/relationship_persistence.rst | 229 ++ doc/build/orm/relationships.rst | 1843 +------------- doc/build/orm/scalar_mapping.rst | 18 + doc/build/orm/self_referential.rst | 261 ++ doc/build/orm/session.rst | 2666 +------------------- doc/build/orm/session_api.rst | 74 + doc/build/orm/session_basics.rst | 744 ++++++ doc/build/orm/session_state_management.rst | 560 ++++ doc/build/orm/session_transaction.rst | 365 +++ doc/build/orm/versioning.rst | 253 ++ doc/build/requirements.txt | 2 +- doc/build/static/detectmobile.js | 7 - doc/build/static/docs.css | 673 ----- doc/build/static/init.js | 44 - doc/build/templates/genindex.mako | 77 - doc/build/templates/layout.mako | 243 -- doc/build/templates/page.mako | 2 - doc/build/templates/search.mako | 21 - doc/build/templates/static_base.mako | 29 - lib/sqlalchemy/dialects/sqlite/base.py | 6 +- lib/sqlalchemy/ext/declarative/__init__.py | 1371 ---------- 80 files changed, 10589 insertions(+), 12209 deletions(-) delete mode 100644 doc/build/builder/__init__.py delete mode 100644 doc/build/builder/autodoc_mods.py delete mode 100644 doc/build/builder/dialect_info.py delete mode 100644 doc/build/builder/mako.py delete mode 100644 doc/build/builder/sqlformatter.py delete mode 100644 doc/build/builder/util.py delete mode 100644 doc/build/builder/viewsource.py create mode 100644 doc/build/core/api_basics.rst create mode 100644 doc/build/core/custom_types.rst create mode 100644 doc/build/core/engines_connections.rst create mode 100644 doc/build/core/type_api.rst create mode 100644 doc/build/core/type_basics.rst delete mode 100644 doc/build/faq.rst create mode 100644 doc/build/faq/connections.rst create mode 100644 doc/build/faq/index.rst create mode 100644 doc/build/faq/metadata_schema.rst create mode 100644 doc/build/faq/ormconfiguration.rst create mode 100644 doc/build/faq/performance.rst create mode 100644 doc/build/faq/sessions.rst create mode 100644 doc/build/faq/sqlexpressions.rst create mode 100644 doc/build/orm/backref.rst create mode 100644 doc/build/orm/basic_relationships.rst create mode 100644 doc/build/orm/cascades.rst create mode 100644 doc/build/orm/classical.rst create mode 100644 doc/build/orm/composites.rst create mode 100644 doc/build/orm/constructors.rst create mode 100644 doc/build/orm/contextual.rst create mode 100644 doc/build/orm/extending.rst delete mode 100644 doc/build/orm/extensions/declarative.rst create mode 100644 doc/build/orm/extensions/declarative/api.rst create mode 100644 doc/build/orm/extensions/declarative/basic_use.rst create mode 100644 doc/build/orm/extensions/declarative/index.rst create mode 100644 doc/build/orm/extensions/declarative/inheritance.rst create mode 100644 doc/build/orm/extensions/declarative/mixins.rst create mode 100644 doc/build/orm/extensions/declarative/relationships.rst create mode 100644 doc/build/orm/extensions/declarative/table_config.rst create mode 100644 doc/build/orm/join_conditions.rst delete mode 100644 doc/build/orm/loading.rst create mode 100644 doc/build/orm/loading_columns.rst create mode 100644 doc/build/orm/loading_objects.rst create mode 100644 doc/build/orm/loading_relationships.rst create mode 100644 doc/build/orm/mapped_attributes.rst create mode 100644 doc/build/orm/mapped_sql_expr.rst create mode 100644 doc/build/orm/mapping_api.rst create mode 100644 doc/build/orm/mapping_columns.rst create mode 100644 doc/build/orm/nonstandard_mappings.rst create mode 100644 doc/build/orm/persistence_techniques.rst create mode 100644 doc/build/orm/relationship_api.rst create mode 100644 doc/build/orm/relationship_persistence.rst create mode 100644 doc/build/orm/scalar_mapping.rst create mode 100644 doc/build/orm/self_referential.rst create mode 100644 doc/build/orm/session_api.rst create mode 100644 doc/build/orm/session_basics.rst create mode 100644 doc/build/orm/session_state_management.rst create mode 100644 doc/build/orm/session_transaction.rst create mode 100644 doc/build/orm/versioning.rst delete mode 100644 doc/build/static/detectmobile.js delete mode 100644 doc/build/static/docs.css delete mode 100644 doc/build/static/init.js delete mode 100644 doc/build/templates/genindex.mako delete mode 100644 doc/build/templates/layout.mako delete mode 100644 doc/build/templates/page.mako delete mode 100644 doc/build/templates/search.mako delete mode 100644 doc/build/templates/static_base.mako diff --git a/doc/build/builder/__init__.py b/doc/build/builder/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/build/builder/autodoc_mods.py b/doc/build/builder/autodoc_mods.py deleted file mode 100644 index 5a6e991bd..000000000 --- a/doc/build/builder/autodoc_mods.py +++ /dev/null @@ -1,102 +0,0 @@ -import re - -def autodoc_skip_member(app, what, name, obj, skip, options): - if what == 'class' and skip and \ - name in ('__init__', '__eq__', '__ne__', '__lt__', - '__le__', '__call__') and \ - obj.__doc__: - return False - else: - return skip - - -_convert_modname = { - "sqlalchemy.sql.sqltypes": "sqlalchemy.types", - "sqlalchemy.sql.type_api": "sqlalchemy.types", - "sqlalchemy.sql.schema": "sqlalchemy.schema", - "sqlalchemy.sql.elements": "sqlalchemy.sql.expression", - "sqlalchemy.sql.selectable": "sqlalchemy.sql.expression", - "sqlalchemy.sql.dml": "sqlalchemy.sql.expression", - "sqlalchemy.sql.ddl": "sqlalchemy.schema", - "sqlalchemy.sql.base": "sqlalchemy.sql.expression" -} - -_convert_modname_w_class = { - ("sqlalchemy.engine.interfaces", "Connectable"): "sqlalchemy.engine", - ("sqlalchemy.sql.base", "DialectKWArgs"): "sqlalchemy.sql.base", -} - -def _adjust_rendered_mod_name(modname, objname): - if (modname, objname) in _convert_modname_w_class: - return _convert_modname_w_class[(modname, objname)] - elif modname in _convert_modname: - return _convert_modname[modname] - else: - return modname - -# im sure this is in the app somewhere, but I don't really -# know where, so we're doing it here. -_track_autodoced = {} -_inherited_names = set() -def autodoc_process_docstring(app, what, name, obj, options, lines): - if what == "class": - _track_autodoced[name] = obj - - # need to translate module names for bases, others - # as we document lots of symbols in namespace modules - # outside of their source - bases = [] - for base in obj.__bases__: - if base is not object: - bases.append(":class:`%s.%s`" % ( - _adjust_rendered_mod_name(base.__module__, base.__name__), - base.__name__)) - - if bases: - lines[:0] = [ - "Bases: %s" % (", ".join(bases)), - "" - ] - - - elif what in ("attribute", "method") and \ - options.get("inherited-members"): - m = re.match(r'(.*?)\.([\w_]+)$', name) - if m: - clsname, attrname = m.group(1, 2) - if clsname in _track_autodoced: - cls = _track_autodoced[clsname] - for supercls in cls.__mro__: - if attrname in supercls.__dict__: - break - if supercls is not cls: - _inherited_names.add("%s.%s" % (supercls.__module__, supercls.__name__)) - _inherited_names.add("%s.%s.%s" % (supercls.__module__, supercls.__name__, attrname)) - lines[:0] = [ - ".. container:: inherited_member", - "", - " *inherited from the* :%s:`~%s.%s.%s` *%s of* :class:`~%s.%s`" % ( - "attr" if what == "attribute" - else "meth", - _adjust_rendered_mod_name(supercls.__module__, supercls.__name__), - supercls.__name__, - attrname, - what, - _adjust_rendered_mod_name(supercls.__module__, supercls.__name__), - supercls.__name__ - ), - "" - ] - -def missing_reference(app, env, node, contnode): - if node.attributes['reftarget'] in _inherited_names: - return node.children[0] - else: - return None - - -def setup(app): - app.connect('autodoc-skip-member', autodoc_skip_member) - app.connect('autodoc-process-docstring', autodoc_process_docstring) - - app.connect('missing-reference', missing_reference) diff --git a/doc/build/builder/dialect_info.py b/doc/build/builder/dialect_info.py deleted file mode 100644 index 48626393d..000000000 --- a/doc/build/builder/dialect_info.py +++ /dev/null @@ -1,175 +0,0 @@ -import re -from sphinx.util.compat import Directive -from docutils import nodes - -class DialectDirective(Directive): - has_content = True - - _dialects = {} - - def _parse_content(self): - d = {} - d['default'] = self.content[0] - d['text'] = [] - idx = 0 - for line in self.content[1:]: - idx += 1 - m = re.match(r'\:(.+?)\: +(.+)', line) - if m: - attrname, value = m.group(1, 2) - d[attrname] = value - else: - break - d["text"] = self.content[idx + 1:] - return d - - def _dbapi_node(self): - - dialect_name, dbapi_name = self.dialect_name.split("+") - - try: - dialect_directive = self._dialects[dialect_name] - except KeyError: - raise Exception("No .. dialect:: %s directive has been established" - % dialect_name) - - output = [] - - content = self._parse_content() - - parent_section_ref = self.state.parent.children[0]['ids'][0] - self._append_dbapi_bullet(dialect_name, dbapi_name, - content['name'], parent_section_ref) - - p = nodes.paragraph('', '', - nodes.Text( - "Support for the %s database via the %s driver." % ( - dialect_directive.database_name, - content['name'] - ), - "Support for the %s database via the %s driver." % ( - dialect_directive.database_name, - content['name'] - ) - ), - ) - - self.state.nested_parse(content['text'], 0, p) - output.append(p) - - if "url" in content or "driverurl" in content: - sec = nodes.section( - '', - nodes.title("DBAPI", "DBAPI"), - ids=["dialect-%s-%s-url" % (dialect_name, dbapi_name)] - ) - if "url" in content: - text = "Documentation and download information (if applicable) "\ - "for %s is available at:\n" % content["name"] - uri = content['url'] - sec.append( - nodes.paragraph('', '', - nodes.Text(text, text), - nodes.reference('', '', - nodes.Text(uri, uri), - refuri=uri, - ) - ) - ) - if "driverurl" in content: - text = "Drivers for this database are available at:\n" - sec.append( - nodes.paragraph('', '', - nodes.Text(text, text), - nodes.reference('', '', - nodes.Text(content['driverurl'], content['driverurl']), - refuri=content['driverurl'] - ) - ) - ) - output.append(sec) - - - if "connectstring" in content: - sec = nodes.section( - '', - nodes.title("Connecting", "Connecting"), - nodes.paragraph('', '', - nodes.Text("Connect String:", "Connect String:"), - nodes.literal_block(content['connectstring'], - content['connectstring']) - ), - ids=["dialect-%s-%s-connect" % (dialect_name, dbapi_name)] - ) - output.append(sec) - - return output - - def _dialect_node(self): - self._dialects[self.dialect_name] = self - - content = self._parse_content() - self.database_name = content['name'] - - self.bullets = nodes.bullet_list() - text = "The following dialect/DBAPI options are available. "\ - "Please refer to individual DBAPI sections for connect information." - sec = nodes.section('', - nodes.paragraph('', '', - nodes.Text( - "Support for the %s database." % content['name'], - "Support for the %s database." % content['name'] - ), - ), - nodes.title("DBAPI Support", "DBAPI Support"), - nodes.paragraph('', '', - nodes.Text(text, text), - self.bullets - ), - ids=["dialect-%s" % self.dialect_name] - ) - - return [sec] - - def _append_dbapi_bullet(self, dialect_name, dbapi_name, name, idname): - env = self.state.document.settings.env - dialect_directive = self._dialects[dialect_name] - try: - relative_uri = env.app.builder.get_relative_uri(dialect_directive.docname, self.docname) - except: - relative_uri = "" - list_node = nodes.list_item('', - nodes.paragraph('', '', - nodes.reference('', '', - nodes.Text(name, name), - refdocname=self.docname, - refuri= relative_uri + "#" + idname - ), - #nodes.Text(" ", " "), - #nodes.reference('', '', - # nodes.Text("(connectstring)", "(connectstring)"), - # refdocname=self.docname, - # refuri=env.app.builder.get_relative_uri( - # dialect_directive.docname, self.docname) + - ## "#" + ("dialect-%s-%s-connect" % - # (dialect_name, dbapi_name)) - # ) - ) - ) - dialect_directive.bullets.append(list_node) - - def run(self): - env = self.state.document.settings.env - self.docname = env.docname - - self.dialect_name = dialect_name = self.content[0] - - has_dbapi = "+" in dialect_name - if has_dbapi: - return self._dbapi_node() - else: - return self._dialect_node() - -def setup(app): - app.add_directive('dialect', DialectDirective) - diff --git a/doc/build/builder/mako.py b/doc/build/builder/mako.py deleted file mode 100644 index 0367bf018..000000000 --- a/doc/build/builder/mako.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import absolute_import - -from sphinx.application import TemplateBridge -from sphinx.jinja2glue import BuiltinTemplateLoader -from mako.lookup import TemplateLookup -import os - -rtd = os.environ.get('READTHEDOCS', None) == 'True' - -class MakoBridge(TemplateBridge): - def init(self, builder, *args, **kw): - self.jinja2_fallback = BuiltinTemplateLoader() - self.jinja2_fallback.init(builder, *args, **kw) - - builder.config.html_context['release_date'] = builder.config['release_date'] - builder.config.html_context['site_base'] = builder.config['site_base'] - - self.lookup = TemplateLookup(directories=builder.config.templates_path, - #format_exceptions=True, - imports=[ - "from builder import util" - ] - ) - - if rtd: - # RTD layout, imported from sqlalchemy.org - import urllib2 - template = urllib2.urlopen(builder.config['site_base'] + "/docs_adapter.mako").read() - self.lookup.put_string("docs_adapter.mako", template) - - setup_ctx = urllib2.urlopen(builder.config['site_base'] + "/docs_adapter.py").read() - lcls = {} - exec(setup_ctx, lcls) - self.setup_ctx = lcls['setup_context'] - - def setup_ctx(self, context): - pass - - def render(self, template, context): - template = template.replace(".html", ".mako") - context['prevtopic'] = context.pop('prev', None) - context['nexttopic'] = context.pop('next', None) - - # local docs layout - context['rtd'] = False - context['toolbar'] = False - context['base'] = "static_base.mako" - - # override context attributes - self.setup_ctx(context) - - context.setdefault('_', lambda x: x) - return self.lookup.get_template(template).render_unicode(**context) - - def render_string(self, template, context): - # this is used for .js, .css etc. and we don't have - # local copies of that stuff here so use the jinja render. - return self.jinja2_fallback.render_string(template, context) - -def setup(app): - app.config['template_bridge'] = "builder.mako.MakoBridge" - app.add_config_value('release_date', "", 'env') - app.add_config_value('site_base', "", 'env') - app.add_config_value('build_number', "", 'env') - diff --git a/doc/build/builder/sqlformatter.py b/doc/build/builder/sqlformatter.py deleted file mode 100644 index 2d8074900..000000000 --- a/doc/build/builder/sqlformatter.py +++ /dev/null @@ -1,132 +0,0 @@ -from pygments.lexer import RegexLexer, bygroups, using -from pygments.token import Token -from pygments.filter import Filter -from pygments.filter import apply_filters -from pygments.lexers import PythonLexer, PythonConsoleLexer -from sphinx.highlighting import PygmentsBridge -from pygments.formatters import HtmlFormatter, LatexFormatter - -import re - - -def _strip_trailing_whitespace(iter_): - buf = list(iter_) - if buf: - buf[-1] = (buf[-1][0], buf[-1][1].rstrip()) - for t, v in buf: - yield t, v - - -class StripDocTestFilter(Filter): - def filter(self, lexer, stream): - for ttype, value in stream: - if ttype is Token.Comment and re.match(r'#\s*doctest:', value): - continue - yield ttype, value - -class PyConWithSQLLexer(RegexLexer): - name = 'PyCon+SQL' - aliases = ['pycon+sql'] - - flags = re.IGNORECASE | re.DOTALL - - tokens = { - 'root': [ - (r'{sql}', Token.Sql.Link, 'sqlpopup'), - (r'{opensql}', Token.Sql.Open, 'opensqlpopup'), - (r'.*?\n', using(PythonConsoleLexer)) - ], - 'sqlpopup': [ - ( - r'(.*?\n)((?:PRAGMA|BEGIN|SELECT|INSERT|DELETE|ROLLBACK|' - 'COMMIT|ALTER|UPDATE|CREATE|DROP|PRAGMA' - '|DESCRIBE).*?(?:{stop}\n?|$))', - bygroups(using(PythonConsoleLexer), Token.Sql.Popup), - "#pop" - ) - ], - 'opensqlpopup': [ - ( - r'.*?(?:{stop}\n*|$)', - Token.Sql, - "#pop" - ) - ] - } - - -class PythonWithSQLLexer(RegexLexer): - name = 'Python+SQL' - aliases = ['pycon+sql'] - - flags = re.IGNORECASE | re.DOTALL - - tokens = { - 'root': [ - (r'{sql}', Token.Sql.Link, 'sqlpopup'), - (r'{opensql}', Token.Sql.Open, 'opensqlpopup'), - (r'.*?\n', using(PythonLexer)) - ], - 'sqlpopup': [ - ( - r'(.*?\n)((?:PRAGMA|BEGIN|SELECT|INSERT|DELETE|ROLLBACK' - '|COMMIT|ALTER|UPDATE|CREATE|DROP' - '|PRAGMA|DESCRIBE).*?(?:{stop}\n?|$))', - bygroups(using(PythonLexer), Token.Sql.Popup), - "#pop" - ) - ], - 'opensqlpopup': [ - ( - r'.*?(?:{stop}\n*|$)', - Token.Sql, - "#pop" - ) - ] - } - -class PopupSQLFormatter(HtmlFormatter): - def _format_lines(self, tokensource): - buf = [] - for ttype, value in apply_filters(tokensource, [StripDocTestFilter()]): - if ttype in Token.Sql: - for t, v in HtmlFormatter._format_lines(self, iter(buf)): - yield t, v - buf = [] - - if ttype is Token.Sql: - yield 1, "
%s
" % \ - re.sub(r'(?:[{stop}|\n]*)$', '', value) - elif ttype is Token.Sql.Link: - yield 1, "sql" - elif ttype is Token.Sql.Popup: - yield 1, "
%s
" % \ - re.sub(r'(?:[{stop}|\n]*)$', '', value) - else: - buf.append((ttype, value)) - - for t, v in _strip_trailing_whitespace( - HtmlFormatter._format_lines(self, iter(buf))): - yield t, v - -class PopupLatexFormatter(LatexFormatter): - def _filter_tokens(self, tokensource): - for ttype, value in apply_filters(tokensource, [StripDocTestFilter()]): - if ttype in Token.Sql: - if ttype is not Token.Sql.Link and ttype is not Token.Sql.Open: - yield Token.Literal, re.sub(r'{stop}', '', value) - else: - continue - else: - yield ttype, value - - def format(self, tokensource, outfile): - LatexFormatter.format(self, self._filter_tokens(tokensource), outfile) - -def setup(app): - app.add_lexer('pycon+sql', PyConWithSQLLexer()) - app.add_lexer('python+sql', PythonWithSQLLexer()) - - PygmentsBridge.html_formatter = PopupSQLFormatter - PygmentsBridge.latex_formatter = PopupLatexFormatter - diff --git a/doc/build/builder/util.py b/doc/build/builder/util.py deleted file mode 100644 index a9dcff001..000000000 --- a/doc/build/builder/util.py +++ /dev/null @@ -1,12 +0,0 @@ -import re - -def striptags(text): - return re.compile(r'<[^>]*>').sub('', text) - -def go(m): - # .html with no anchor if present, otherwise "#" for top of page - return m.group(1) or '#' - -def strip_toplevel_anchors(text): - return re.compile(r'(\.html)?#[-\w]+-toplevel').sub(go, text) - diff --git a/doc/build/builder/viewsource.py b/doc/build/builder/viewsource.py deleted file mode 100644 index 088cef2c2..000000000 --- a/doc/build/builder/viewsource.py +++ /dev/null @@ -1,209 +0,0 @@ -from docutils import nodes -from sphinx.ext.viewcode import collect_pages -from sphinx.pycode import ModuleAnalyzer -import imp -from sphinx import addnodes -import re -from sphinx.util.compat import Directive -import os -from docutils.statemachine import StringList -from sphinx.environment import NoUri - -import sys - -py2k = sys.version_info < (3, 0) -if py2k: - text_type = unicode -else: - text_type = str - -def view_source(name, rawtext, text, lineno, inliner, - options={}, content=[]): - - env = inliner.document.settings.env - - node = _view_source_node(env, text, None) - return [node], [] - -def _view_source_node(env, text, state): - # pretend we're using viewcode fully, - # install the context it looks for - if not hasattr(env, '_viewcode_modules'): - env._viewcode_modules = {} - - modname = text - text = modname.split(".")[-1] + ".py" - - # imitate sphinx . syntax - if modname.startswith("."): - # see if the modname needs to be corrected in terms - # of current module context - base_module = env.temp_data.get('autodoc:module') - if base_module is None: - base_module = env.temp_data.get('py:module') - - if base_module: - modname = base_module + modname - - urito = env.app.builder.get_relative_uri - - # we're showing code examples which may have dependencies - # which we really don't want to have required so load the - # module by file, not import (though we are importing) - # the top level module here... - pathname = None - for token in modname.split("."): - file_, pathname, desc = imp.find_module(token, [pathname] if pathname else None) - if file_: - file_.close() - - # unlike viewcode which silently traps exceptions, - # I want this to totally barf if the file can't be loaded. - # a failed build better than a complete build missing - # key content - analyzer = ModuleAnalyzer.for_file(pathname, modname) - # copied from viewcode - analyzer.find_tags() - if not isinstance(analyzer.code, text_type): - code = analyzer.code.decode(analyzer.encoding) - else: - code = analyzer.code - - if state is not None: - docstring = _find_mod_docstring(analyzer) - if docstring: - # get rid of "foo.py" at the top - docstring = re.sub(r"^[a-zA-Z_0-9]+\.py", "", docstring) - - # strip - docstring = docstring.strip() - - # yank only first paragraph - docstring = docstring.split("\n\n")[0].strip() - else: - docstring = None - - entry = code, analyzer.tags, {} - env._viewcode_modules[modname] = entry - pagename = '_modules/' + modname.replace('.', '/') - - try: - refuri = urito(env.docname, pagename) - except NoUri: - # if we're in the latex builder etc., this seems - # to be what we get - refuri = None - - - if docstring: - # embed the ref with the doc text so that it isn't - # a separate paragraph - if refuri: - docstring = "`%s <%s>`_ - %s" % (text, refuri, docstring) - else: - docstring = "``%s`` - %s" % (text, docstring) - para = nodes.paragraph('', '') - state.nested_parse(StringList([docstring]), 0, para) - return_node = para - else: - if refuri: - refnode = nodes.reference('', '', - nodes.Text(text, text), - refuri=urito(env.docname, pagename) - ) - else: - refnode = nodes.Text(text, text) - - if state: - return_node = nodes.paragraph('', '', refnode) - else: - return_node = refnode - - return return_node - -from sphinx.pycode.pgen2 import token - -def _find_mod_docstring(analyzer): - """attempt to locate the module-level docstring. - - Note that sphinx autodoc just uses ``__doc__``. But we don't want - to import the module, so we need to parse for it. - - """ - analyzer.tokenize() - for type_, parsed_line, start_pos, end_pos, raw_line in analyzer.tokens: - if type_ == token.COMMENT: - continue - elif type_ == token.STRING: - return eval(parsed_line) - else: - return None - -def _parse_content(content): - d = {} - d['text'] = [] - idx = 0 - for line in content: - idx += 1 - m = re.match(r' *\:(.+?)\:(?: +(.+))?', line) - if m: - attrname, value = m.group(1, 2) - d[attrname] = value or '' - else: - break - d["text"] = content[idx:] - return d - -def _comma_list(text): - return re.split(r"\s*,\s*", text.strip()) - -class AutoSourceDirective(Directive): - has_content = True - - def run(self): - content = _parse_content(self.content) - - - env = self.state.document.settings.env - self.docname = env.docname - - sourcefile = self.state.document.current_source.split(os.pathsep)[0] - dir_ = os.path.dirname(sourcefile) - files = [ - f for f in os.listdir(dir_) if f.endswith(".py") - and f != "__init__.py" - ] - - if "files" in content: - # ordered listing of files to include - files = [fname for fname in _comma_list(content["files"]) - if fname in set(files)] - - node = nodes.paragraph('', '', - nodes.Text("Listing of files:", "Listing of files:") - ) - - bullets = nodes.bullet_list() - for fname in files: - modname, ext = os.path.splitext(fname) - # relative lookup - modname = "." + modname - - link = _view_source_node(env, modname, self.state) - - list_node = nodes.list_item('', - link - ) - bullets += list_node - - node += bullets - - return [node] - -def setup(app): - app.add_role('viewsource', view_source) - - app.add_directive('autosource', AutoSourceDirective) - - # from sphinx.ext.viewcode - app.connect('html-collect-pages', collect_pages) diff --git a/doc/build/conf.py b/doc/build/conf.py index 5277134e7..7e17fcd59 100644 --- a/doc/build/conf.py +++ b/doc/build/conf.py @@ -34,13 +34,9 @@ import sqlalchemy extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', - 'builder.autodoc_mods', + 'zzzeeksphinx', 'changelog', 'sphinx_paramlinks', - 'builder.dialect_info', - 'builder.mako', - 'builder.sqlformatter', - 'builder.viewsource', ] # Add any paths that contain templates here, relative to this directory. @@ -74,6 +70,21 @@ changelog_render_pullreq = { changelog_render_changeset = "http://www.sqlalchemy.org/trac/changeset/%s" +autodocmods_convert_modname = { + "sqlalchemy.sql.sqltypes": "sqlalchemy.types", + "sqlalchemy.sql.type_api": "sqlalchemy.types", + "sqlalchemy.sql.schema": "sqlalchemy.schema", + "sqlalchemy.sql.elements": "sqlalchemy.sql.expression", + "sqlalchemy.sql.selectable": "sqlalchemy.sql.expression", + "sqlalchemy.sql.dml": "sqlalchemy.sql.expression", + "sqlalchemy.sql.ddl": "sqlalchemy.schema", + "sqlalchemy.sql.base": "sqlalchemy.sql.expression" +} + +autodocmods_convert_modname_w_class = { + ("sqlalchemy.engine.interfaces", "Connectable"): "sqlalchemy.engine", + ("sqlalchemy.sql.base", "DialectKWArgs"): "sqlalchemy.sql.base", +} # The encoding of source files. #source_encoding = 'utf-8-sig' @@ -97,6 +108,8 @@ release = "1.0.0" release_date = "Not released" site_base = os.environ.get("RTD_SITE_BASE", "http://www.sqlalchemy.org") +site_adapter_template = "docs_adapter.mako" +site_adapter_py = "docs_adapter.py" # arbitrary number recognized by builders.py, incrementing this # will force a rebuild @@ -144,7 +157,7 @@ gettext_compact = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'zzzeeksphinx' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/build/contents.rst b/doc/build/contents.rst index df80e9b79..95b5e9a19 100644 --- a/doc/build/contents.rst +++ b/doc/build/contents.rst @@ -13,6 +13,7 @@ documentation, see :ref:`index_toplevel`. orm/index core/index dialects/index + faq/index changelog/index Indices and tables diff --git a/doc/build/core/api_basics.rst b/doc/build/core/api_basics.rst new file mode 100644 index 000000000..e56a1117b --- /dev/null +++ b/doc/build/core/api_basics.rst @@ -0,0 +1,12 @@ +================= +Core API Basics +================= + +.. toctree:: + :maxdepth: 2 + + event + inspection + interfaces + exceptions + internals diff --git a/doc/build/core/custom_types.rst b/doc/build/core/custom_types.rst new file mode 100644 index 000000000..92c5ca6cf --- /dev/null +++ b/doc/build/core/custom_types.rst @@ -0,0 +1,498 @@ +.. _types_custom: + +Custom Types +------------ + +A variety of methods exist to redefine the behavior of existing types +as well as to provide new ones. + +Overriding Type Compilation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A frequent need is to force the "string" version of a type, that is +the one rendered in a CREATE TABLE statement or other SQL function +like CAST, to be changed. For example, an application may want +to force the rendering of ``BINARY`` for all platforms +except for one, in which is wants ``BLOB`` to be rendered. Usage +of an existing generic type, in this case :class:`.LargeBinary`, is +preferred for most use cases. But to control +types more accurately, a compilation directive that is per-dialect +can be associated with any type:: + + from sqlalchemy.ext.compiler import compiles + from sqlalchemy.types import BINARY + + @compiles(BINARY, "sqlite") + def compile_binary_sqlite(type_, compiler, **kw): + return "BLOB" + +The above code allows the usage of :class:`.types.BINARY`, which +will produce the string ``BINARY`` against all backends except SQLite, +in which case it will produce ``BLOB``. + +See the section :ref:`type_compilation_extension`, a subsection of +:ref:`sqlalchemy.ext.compiler_toplevel`, for additional examples. + +.. _types_typedecorator: + +Augmenting Existing Types +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :class:`.TypeDecorator` allows the creation of custom types which +add bind-parameter and result-processing behavior to an existing +type object. It is used when additional in-Python marshaling of data +to and from the database is required. + +.. note:: + + The bind- and result-processing of :class:`.TypeDecorator` + is *in addition* to the processing already performed by the hosted + type, which is customized by SQLAlchemy on a per-DBAPI basis to perform + processing specific to that DBAPI. To change the DBAPI-level processing + for an existing type, see the section :ref:`replacing_processors`. + +.. autoclass:: TypeDecorator + :members: + :inherited-members: + + +TypeDecorator Recipes +~~~~~~~~~~~~~~~~~~~~~ +A few key :class:`.TypeDecorator` recipes follow. + +.. _coerce_to_unicode: + +Coercing Encoded Strings to Unicode +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A common source of confusion regarding the :class:`.Unicode` type +is that it is intended to deal *only* with Python ``unicode`` objects +on the Python side, meaning values passed to it as bind parameters +must be of the form ``u'some string'`` if using Python 2 and not 3. +The encoding/decoding functions it performs are only to suit what the +DBAPI in use requires, and are primarily a private implementation detail. + +The use case of a type that can safely receive Python bytestrings, +that is strings that contain non-ASCII characters and are not ``u''`` +objects in Python 2, can be achieved using a :class:`.TypeDecorator` +which coerces as needed:: + + from sqlalchemy.types import TypeDecorator, Unicode + + class CoerceUTF8(TypeDecorator): + """Safely coerce Python bytestrings to Unicode + before passing off to the database.""" + + impl = Unicode + + def process_bind_param(self, value, dialect): + if isinstance(value, str): + value = value.decode('utf-8') + return value + +Rounding Numerics +^^^^^^^^^^^^^^^^^ + +Some database connectors like those of SQL Server choke if a Decimal is passed with too +many decimal places. Here's a recipe that rounds them down:: + + from sqlalchemy.types import TypeDecorator, Numeric + from decimal import Decimal + + class SafeNumeric(TypeDecorator): + """Adds quantization to Numeric.""" + + impl = Numeric + + def __init__(self, *arg, **kw): + TypeDecorator.__init__(self, *arg, **kw) + self.quantize_int = -(self.impl.precision - self.impl.scale) + self.quantize = Decimal(10) ** self.quantize_int + + def process_bind_param(self, value, dialect): + if isinstance(value, Decimal) and \ + value.as_tuple()[2] < self.quantize_int: + value = value.quantize(self.quantize) + return value + +.. _custom_guid_type: + +Backend-agnostic GUID Type +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Receives and returns Python uuid() objects. Uses the PG UUID type +when using Postgresql, CHAR(32) on other backends, storing them +in stringified hex format. Can be modified to store +binary in CHAR(16) if desired:: + + from sqlalchemy.types import TypeDecorator, CHAR + from sqlalchemy.dialects.postgresql import UUID + import uuid + + class GUID(TypeDecorator): + """Platform-independent GUID type. + + Uses Postgresql's UUID type, otherwise uses + CHAR(32), storing as stringified hex values. + + """ + impl = CHAR + + def load_dialect_impl(self, dialect): + if dialect.name == 'postgresql': + return dialect.type_descriptor(UUID()) + else: + return dialect.type_descriptor(CHAR(32)) + + def process_bind_param(self, value, dialect): + if value is None: + return value + elif dialect.name == 'postgresql': + return str(value) + else: + if not isinstance(value, uuid.UUID): + return "%.32x" % uuid.UUID(value) + else: + # hexstring + return "%.32x" % value + + def process_result_value(self, value, dialect): + if value is None: + return value + else: + return uuid.UUID(value) + +Marshal JSON Strings +^^^^^^^^^^^^^^^^^^^^^ + +This type uses ``simplejson`` to marshal Python data structures +to/from JSON. Can be modified to use Python's builtin json encoder:: + + from sqlalchemy.types import TypeDecorator, VARCHAR + import json + + class JSONEncodedDict(TypeDecorator): + """Represents an immutable structure as a json-encoded string. + + Usage:: + + JSONEncodedDict(255) + + """ + + impl = VARCHAR + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + +Note that the ORM by default will not detect "mutability" on such a type - +meaning, in-place changes to values will not be detected and will not be +flushed. Without further steps, you instead would need to replace the existing +value with a new one on each parent object to detect changes. Note that +there's nothing wrong with this, as many applications may not require that the +values are ever mutated once created. For those which do have this requirement, +support for mutability is best applied using the ``sqlalchemy.ext.mutable`` +extension - see the example in :ref:`mutable_toplevel`. + +.. _replacing_processors: + +Replacing the Bind/Result Processing of Existing Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most augmentation of type behavior at the bind/result level +is achieved using :class:`.TypeDecorator`. For the rare scenario +where the specific processing applied by SQLAlchemy at the DBAPI +level needs to be replaced, the SQLAlchemy type can be subclassed +directly, and the ``bind_processor()`` or ``result_processor()`` +methods can be overridden. Doing so requires that the +``adapt()`` method also be overridden. This method is the mechanism +by which SQLAlchemy produces DBAPI-specific type behavior during +statement execution. Overriding it allows a copy of the custom +type to be used in lieu of a DBAPI-specific type. Below we subclass +the :class:`.types.TIME` type to have custom result processing behavior. +The ``process()`` function will receive ``value`` from the DBAPI +cursor directly:: + + class MySpecialTime(TIME): + def __init__(self, special_argument): + super(MySpecialTime, self).__init__() + self.special_argument = special_argument + + def result_processor(self, dialect, coltype): + import datetime + time = datetime.time + def process(value): + if value is not None: + microseconds = value.microseconds + seconds = value.seconds + minutes = seconds / 60 + return time( + minutes / 60, + minutes % 60, + seconds - minutes * 60, + microseconds) + else: + return None + return process + + def adapt(self, impltype): + return MySpecialTime(self.special_argument) + +.. _types_sql_value_processing: + +Applying SQL-level Bind/Result Processing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As seen in the sections :ref:`types_typedecorator` and :ref:`replacing_processors`, +SQLAlchemy allows Python functions to be invoked both when parameters are sent +to a statement, as well as when result rows are loaded from the database, to apply +transformations to the values as they are sent to or from the database. It is also +possible to define SQL-level transformations as well. The rationale here is when +only the relational database contains a particular series of functions that are necessary +to coerce incoming and outgoing data between an application and persistence format. +Examples include using database-defined encryption/decryption functions, as well +as stored procedures that handle geographic data. The Postgis extension to Postgresql +includes an extensive array of SQL functions that are necessary for coercing +data into particular formats. + +Any :class:`.TypeEngine`, :class:`.UserDefinedType` or :class:`.TypeDecorator` subclass +can include implementations of +:meth:`.TypeEngine.bind_expression` and/or :meth:`.TypeEngine.column_expression`, which +when defined to return a non-``None`` value should return a :class:`.ColumnElement` +expression to be injected into the SQL statement, either surrounding +bound parameters or a column expression. For example, to build a ``Geometry`` +type which will apply the Postgis function ``ST_GeomFromText`` to all outgoing +values and the function ``ST_AsText`` to all incoming data, we can create +our own subclass of :class:`.UserDefinedType` which provides these methods +in conjunction with :data:`~.sqlalchemy.sql.expression.func`:: + + from sqlalchemy import func + from sqlalchemy.types import UserDefinedType + + class Geometry(UserDefinedType): + def get_col_spec(self): + return "GEOMETRY" + + def bind_expression(self, bindvalue): + return func.ST_GeomFromText(bindvalue, type_=self) + + def column_expression(self, col): + return func.ST_AsText(col, type_=self) + +We can apply the ``Geometry`` type into :class:`.Table` metadata +and use it in a :func:`.select` construct:: + + geometry = Table('geometry', metadata, + Column('geom_id', Integer, primary_key=True), + Column('geom_data', Geometry) + ) + + print select([geometry]).where( + geometry.c.geom_data == 'LINESTRING(189412 252431,189631 259122)') + +The resulting SQL embeds both functions as appropriate. ``ST_AsText`` +is applied to the columns clause so that the return value is run through +the function before passing into a result set, and ``ST_GeomFromText`` +is run on the bound parameter so that the passed-in value is converted:: + + SELECT geometry.geom_id, ST_AsText(geometry.geom_data) AS geom_data_1 + FROM geometry + WHERE geometry.geom_data = ST_GeomFromText(:geom_data_2) + +The :meth:`.TypeEngine.column_expression` method interacts with the +mechanics of the compiler such that the SQL expression does not interfere +with the labeling of the wrapped expression. Such as, if we rendered +a :func:`.select` against a :func:`.label` of our expression, the string +label is moved to the outside of the wrapped expression:: + + print select([geometry.c.geom_data.label('my_data')]) + +Output:: + + SELECT ST_AsText(geometry.geom_data) AS my_data + FROM geometry + +For an example of subclassing a built in type directly, we subclass +:class:`.postgresql.BYTEA` to provide a ``PGPString``, which will make use of the +Postgresql ``pgcrypto`` extension to encrpyt/decrypt values +transparently:: + + from sqlalchemy import create_engine, String, select, func, \ + MetaData, Table, Column, type_coerce + + from sqlalchemy.dialects.postgresql import BYTEA + + class PGPString(BYTEA): + def __init__(self, passphrase, length=None): + super(PGPString, self).__init__(length) + self.passphrase = passphrase + + def bind_expression(self, bindvalue): + # convert the bind's type from PGPString to + # String, so that it's passed to psycopg2 as is without + # a dbapi.Binary wrapper + bindvalue = type_coerce(bindvalue, String) + return func.pgp_sym_encrypt(bindvalue, self.passphrase) + + def column_expression(self, col): + return func.pgp_sym_decrypt(col, self.passphrase) + + metadata = MetaData() + message = Table('message', metadata, + Column('username', String(50)), + Column('message', + PGPString("this is my passphrase", length=1000)), + ) + + engine = create_engine("postgresql://scott:tiger@localhost/test", echo=True) + with engine.begin() as conn: + metadata.create_all(conn) + + conn.execute(message.insert(), username="some user", + message="this is my message") + + print conn.scalar( + select([message.c.message]).\ + where(message.c.username == "some user") + ) + +The ``pgp_sym_encrypt`` and ``pgp_sym_decrypt`` functions are applied +to the INSERT and SELECT statements:: + + INSERT INTO message (username, message) + VALUES (%(username)s, pgp_sym_encrypt(%(message)s, %(pgp_sym_encrypt_1)s)) + {'username': 'some user', 'message': 'this is my message', + 'pgp_sym_encrypt_1': 'this is my passphrase'} + + SELECT pgp_sym_decrypt(message.message, %(pgp_sym_decrypt_1)s) AS message_1 + FROM message + WHERE message.username = %(username_1)s + {'pgp_sym_decrypt_1': 'this is my passphrase', 'username_1': 'some user'} + + +.. versionadded:: 0.8 Added the :meth:`.TypeEngine.bind_expression` and + :meth:`.TypeEngine.column_expression` methods. + +See also: + +:ref:`examples_postgis` + +.. _types_operators: + +Redefining and Creating New Operators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SQLAlchemy Core defines a fixed set of expression operators available to all column expressions. +Some of these operations have the effect of overloading Python's built in operators; +examples of such operators include +:meth:`.ColumnOperators.__eq__` (``table.c.somecolumn == 'foo'``), +:meth:`.ColumnOperators.__invert__` (``~table.c.flag``), +and :meth:`.ColumnOperators.__add__` (``table.c.x + table.c.y``). Other operators are exposed as +explicit methods on column expressions, such as +:meth:`.ColumnOperators.in_` (``table.c.value.in_(['x', 'y'])``) and :meth:`.ColumnOperators.like` +(``table.c.value.like('%ed%')``). + +The Core expression constructs in all cases consult the type of the expression in order to determine +the behavior of existing operators, as well as to locate additional operators that aren't part of +the built in set. The :class:`.TypeEngine` base class defines a root "comparison" implementation +:class:`.TypeEngine.Comparator`, and many specific types provide their own sub-implementations of this +class. User-defined :class:`.TypeEngine.Comparator` implementations can be built directly into a +simple subclass of a particular type in order to override or define new operations. Below, +we create a :class:`.Integer` subclass which overrides the :meth:`.ColumnOperators.__add__` operator:: + + from sqlalchemy import Integer + + class MyInt(Integer): + class comparator_factory(Integer.Comparator): + def __add__(self, other): + return self.op("goofy")(other) + +The above configuration creates a new class ``MyInt``, which +establishes the :attr:`.TypeEngine.comparator_factory` attribute as +referring to a new class, subclassing the :class:`.TypeEngine.Comparator` class +associated with the :class:`.Integer` type. + +Usage:: + + >>> sometable = Table("sometable", metadata, Column("data", MyInt)) + >>> print sometable.c.data + 5 + sometable.data goofy :data_1 + +The implementation for :meth:`.ColumnOperators.__add__` is consulted +by an owning SQL expression, by instantiating the :class:`.TypeEngine.Comparator` with +itself as the ``expr`` attribute. The mechanics of the expression +system are such that operations continue recursively until an +expression object produces a new SQL expression construct. Above, we +could just as well have said ``self.expr.op("goofy")(other)`` instead +of ``self.op("goofy")(other)``. + +New methods added to a :class:`.TypeEngine.Comparator` are exposed on an +owning SQL expression +using a ``__getattr__`` scheme, which exposes methods added to +:class:`.TypeEngine.Comparator` onto the owning :class:`.ColumnElement`. +For example, to add a ``log()`` function +to integers:: + + from sqlalchemy import Integer, func + + class MyInt(Integer): + class comparator_factory(Integer.Comparator): + def log(self, other): + return func.log(self.expr, other) + +Using the above type:: + + >>> print sometable.c.data.log(5) + log(:log_1, :log_2) + + +Unary operations +are also possible. For example, to add an implementation of the +Postgresql factorial operator, we combine the :class:`.UnaryExpression` construct +along with a :class:`.custom_op` to produce the factorial expression:: + + from sqlalchemy import Integer + from sqlalchemy.sql.expression import UnaryExpression + from sqlalchemy.sql import operators + + class MyInteger(Integer): + class comparator_factory(Integer.Comparator): + def factorial(self): + return UnaryExpression(self.expr, + modifier=operators.custom_op("!"), + type_=MyInteger) + +Using the above type:: + + >>> from sqlalchemy.sql import column + >>> print column('x', MyInteger).factorial() + x ! + +See also: + +:attr:`.TypeEngine.comparator_factory` + +.. versionadded:: 0.8 The expression system was enhanced to support + customization of operators on a per-type level. + + +Creating New Types +~~~~~~~~~~~~~~~~~~ + +The :class:`.UserDefinedType` class is provided as a simple base class +for defining entirely new database types. Use this to represent native +database types not known by SQLAlchemy. If only Python translation behavior +is needed, use :class:`.TypeDecorator` instead. + +.. autoclass:: UserDefinedType + :members: + + diff --git a/doc/build/core/engines_connections.rst b/doc/build/core/engines_connections.rst new file mode 100644 index 000000000..f163a7629 --- /dev/null +++ b/doc/build/core/engines_connections.rst @@ -0,0 +1,11 @@ +========================= +Engine and Connection Use +========================= + +.. toctree:: + :maxdepth: 2 + + engines + connections + pooling + events diff --git a/doc/build/core/expression_api.rst b/doc/build/core/expression_api.rst index 99bb98881..b32fa0e23 100644 --- a/doc/build/core/expression_api.rst +++ b/doc/build/core/expression_api.rst @@ -16,5 +16,5 @@ see :ref:`sqlexpression_toplevel`. selectable dml functions - types - + compiler + serializer diff --git a/doc/build/core/index.rst b/doc/build/core/index.rst index 210f28412..26c26af07 100644 --- a/doc/build/core/index.rst +++ b/doc/build/core/index.rst @@ -9,19 +9,11 @@ In contrast to the ORM’s domain-centric mode of usage, the SQL Expression Language provides a schema-centric usage paradigm. .. toctree:: - :maxdepth: 3 + :maxdepth: 2 tutorial expression_api schema - engines - connections - pooling - event - events - compiler - inspection - serializer - interfaces - exceptions - internals + types + engines_connections + api_basics diff --git a/doc/build/core/schema.rst b/doc/build/core/schema.rst index aeb04be18..8553ebcbf 100644 --- a/doc/build/core/schema.rst +++ b/doc/build/core/schema.rst @@ -33,7 +33,7 @@ real DDL. They are therefore most intuitive to those who have some background in creating real schema generation scripts. .. toctree:: - :maxdepth: 1 + :maxdepth: 2 metadata reflection @@ -41,5 +41,3 @@ in creating real schema generation scripts. constraints ddl - - diff --git a/doc/build/core/type_api.rst b/doc/build/core/type_api.rst new file mode 100644 index 000000000..88da4939e --- /dev/null +++ b/doc/build/core/type_api.rst @@ -0,0 +1,22 @@ +.. module:: sqlalchemy.types + +.. _types_api: + +Base Type API +-------------- + +.. autoclass:: TypeEngine + :members: + + +.. autoclass:: Concatenable + :members: + :inherited-members: + + +.. autoclass:: NullType + + +.. autoclass:: Variant + + :members: with_variant, __init__ diff --git a/doc/build/core/type_basics.rst b/doc/build/core/type_basics.rst new file mode 100644 index 000000000..1ff1baac2 --- /dev/null +++ b/doc/build/core/type_basics.rst @@ -0,0 +1,229 @@ +Column and Data Types +===================== + +.. module:: sqlalchemy.types + +SQLAlchemy provides abstractions for most common database data types, +and a mechanism for specifying your own custom data types. + +The methods and attributes of type objects are rarely used directly. +Type objects are supplied to :class:`~sqlalchemy.schema.Table` definitions +and can be supplied as type hints to `functions` for occasions where +the database driver returns an incorrect type. + +.. code-block:: pycon + + >>> users = Table('users', metadata, + ... Column('id', Integer, primary_key=True) + ... Column('login', String(32)) + ... ) + + +SQLAlchemy will use the ``Integer`` and ``String(32)`` type +information when issuing a ``CREATE TABLE`` statement and will use it +again when reading back rows ``SELECTed`` from the database. +Functions that accept a type (such as :func:`~sqlalchemy.schema.Column`) will +typically accept a type class or instance; ``Integer`` is equivalent +to ``Integer()`` with no construction arguments in this case. + +.. _types_generic: + +Generic Types +------------- + +Generic types specify a column that can read, write and store a +particular type of Python data. SQLAlchemy will choose the best +database column type available on the target database when issuing a +``CREATE TABLE`` statement. For complete control over which column +type is emitted in ``CREATE TABLE``, such as ``VARCHAR`` see `SQL +Standard Types`_ and the other sections of this chapter. + +.. autoclass:: BigInteger + :members: + +.. autoclass:: Boolean + :members: + +.. autoclass:: Date + :members: + +.. autoclass:: DateTime + :members: + +.. autoclass:: Enum + :members: __init__, create, drop + +.. autoclass:: Float + :members: + +.. autoclass:: Integer + :members: + +.. autoclass:: Interval + :members: + +.. autoclass:: LargeBinary + :members: + +.. autoclass:: MatchType + :members: + +.. autoclass:: Numeric + :members: + +.. autoclass:: PickleType + :members: + +.. autoclass:: SchemaType + :members: + :undoc-members: + +.. autoclass:: SmallInteger + :members: + +.. autoclass:: String + :members: + +.. autoclass:: Text + :members: + +.. autoclass:: Time + :members: + +.. autoclass:: Unicode + :members: + +.. autoclass:: UnicodeText + :members: + +.. _types_sqlstandard: + +SQL Standard Types +------------------ + +The SQL standard types always create database column types of the same +name when ``CREATE TABLE`` is issued. Some types may not be supported +on all databases. + +.. autoclass:: BIGINT + + +.. autoclass:: BINARY + + +.. autoclass:: BLOB + + +.. autoclass:: BOOLEAN + + +.. autoclass:: CHAR + + +.. autoclass:: CLOB + + +.. autoclass:: DATE + + +.. autoclass:: DATETIME + + +.. autoclass:: DECIMAL + + +.. autoclass:: FLOAT + + +.. autoclass:: INT + + +.. autoclass:: sqlalchemy.types.INTEGER + + +.. autoclass:: NCHAR + + +.. autoclass:: NVARCHAR + + +.. autoclass:: NUMERIC + + +.. autoclass:: REAL + + +.. autoclass:: SMALLINT + + +.. autoclass:: TEXT + + +.. autoclass:: TIME + + +.. autoclass:: TIMESTAMP + + +.. autoclass:: VARBINARY + + +.. autoclass:: VARCHAR + + +.. _types_vendor: + +Vendor-Specific Types +--------------------- + +Database-specific types are also available for import from each +database's dialect module. See the :ref:`dialect_toplevel` +reference for the database you're interested in. + +For example, MySQL has a ``BIGINT`` type and PostgreSQL has an +``INET`` type. To use these, import them from the module explicitly:: + + from sqlalchemy.dialects import mysql + + table = Table('foo', metadata, + Column('id', mysql.BIGINT), + Column('enumerates', mysql.ENUM('a', 'b', 'c')) + ) + +Or some PostgreSQL types:: + + from sqlalchemy.dialects import postgresql + + table = Table('foo', metadata, + Column('ipaddress', postgresql.INET), + Column('elements', postgresql.ARRAY(String)) + ) + +Each dialect provides the full set of typenames supported by +that backend within its `__all__` collection, so that a simple +`import *` or similar will import all supported types as +implemented for that backend:: + + from sqlalchemy.dialects.postgresql import * + + t = Table('mytable', metadata, + Column('id', INTEGER, primary_key=True), + Column('name', VARCHAR(300)), + Column('inetaddr', INET) + ) + +Where above, the INTEGER and VARCHAR types are ultimately from +sqlalchemy.types, and INET is specific to the Postgresql dialect. + +Some dialect level types have the same name as the SQL standard type, +but also provide additional arguments. For example, MySQL implements +the full range of character and string types including additional arguments +such as `collation` and `charset`:: + + from sqlalchemy.dialects.mysql import VARCHAR, TEXT + + table = Table('foo', meta, + Column('col1', VARCHAR(200, collation='binary')), + Column('col2', TEXT(charset='latin1')) + ) + diff --git a/doc/build/core/types.rst b/doc/build/core/types.rst index 22b36a648..9d2b66124 100644 --- a/doc/build/core/types.rst +++ b/doc/build/core/types.rst @@ -3,747 +3,9 @@ Column and Data Types ===================== -.. module:: sqlalchemy.types +.. toctree:: + :maxdepth: 2 -SQLAlchemy provides abstractions for most common database data types, -and a mechanism for specifying your own custom data types. - -The methods and attributes of type objects are rarely used directly. -Type objects are supplied to :class:`~sqlalchemy.schema.Table` definitions -and can be supplied as type hints to `functions` for occasions where -the database driver returns an incorrect type. - -.. code-block:: pycon - - >>> users = Table('users', metadata, - ... Column('id', Integer, primary_key=True) - ... Column('login', String(32)) - ... ) - - -SQLAlchemy will use the ``Integer`` and ``String(32)`` type -information when issuing a ``CREATE TABLE`` statement and will use it -again when reading back rows ``SELECTed`` from the database. -Functions that accept a type (such as :func:`~sqlalchemy.schema.Column`) will -typically accept a type class or instance; ``Integer`` is equivalent -to ``Integer()`` with no construction arguments in this case. - -.. _types_generic: - -Generic Types -------------- - -Generic types specify a column that can read, write and store a -particular type of Python data. SQLAlchemy will choose the best -database column type available on the target database when issuing a -``CREATE TABLE`` statement. For complete control over which column -type is emitted in ``CREATE TABLE``, such as ``VARCHAR`` see `SQL -Standard Types`_ and the other sections of this chapter. - -.. autoclass:: BigInteger - :members: - -.. autoclass:: Boolean - :members: - -.. autoclass:: Date - :members: - -.. autoclass:: DateTime - :members: - -.. autoclass:: Enum - :members: __init__, create, drop - -.. autoclass:: Float - :members: - -.. autoclass:: Integer - :members: - -.. autoclass:: Interval - :members: - -.. autoclass:: LargeBinary - :members: - -.. autoclass:: MatchType - :members: - -.. autoclass:: Numeric - :members: - -.. autoclass:: PickleType - :members: - -.. autoclass:: SchemaType - :members: - :undoc-members: - -.. autoclass:: SmallInteger - :members: - -.. autoclass:: String - :members: - -.. autoclass:: Text - :members: - -.. autoclass:: Time - :members: - -.. autoclass:: Unicode - :members: - -.. autoclass:: UnicodeText - :members: - -.. _types_sqlstandard: - -SQL Standard Types ------------------- - -The SQL standard types always create database column types of the same -name when ``CREATE TABLE`` is issued. Some types may not be supported -on all databases. - -.. autoclass:: BIGINT - - -.. autoclass:: BINARY - - -.. autoclass:: BLOB - - -.. autoclass:: BOOLEAN - - -.. autoclass:: CHAR - - -.. autoclass:: CLOB - - -.. autoclass:: DATE - - -.. autoclass:: DATETIME - - -.. autoclass:: DECIMAL - - -.. autoclass:: FLOAT - - -.. autoclass:: INT - - -.. autoclass:: sqlalchemy.types.INTEGER - - -.. autoclass:: NCHAR - - -.. autoclass:: NVARCHAR - - -.. autoclass:: NUMERIC - - -.. autoclass:: REAL - - -.. autoclass:: SMALLINT - - -.. autoclass:: TEXT - - -.. autoclass:: TIME - - -.. autoclass:: TIMESTAMP - - -.. autoclass:: VARBINARY - - -.. autoclass:: VARCHAR - - -.. _types_vendor: - -Vendor-Specific Types ---------------------- - -Database-specific types are also available for import from each -database's dialect module. See the :ref:`dialect_toplevel` -reference for the database you're interested in. - -For example, MySQL has a ``BIGINT`` type and PostgreSQL has an -``INET`` type. To use these, import them from the module explicitly:: - - from sqlalchemy.dialects import mysql - - table = Table('foo', metadata, - Column('id', mysql.BIGINT), - Column('enumerates', mysql.ENUM('a', 'b', 'c')) - ) - -Or some PostgreSQL types:: - - from sqlalchemy.dialects import postgresql - - table = Table('foo', metadata, - Column('ipaddress', postgresql.INET), - Column('elements', postgresql.ARRAY(String)) - ) - -Each dialect provides the full set of typenames supported by -that backend within its `__all__` collection, so that a simple -`import *` or similar will import all supported types as -implemented for that backend:: - - from sqlalchemy.dialects.postgresql import * - - t = Table('mytable', metadata, - Column('id', INTEGER, primary_key=True), - Column('name', VARCHAR(300)), - Column('inetaddr', INET) - ) - -Where above, the INTEGER and VARCHAR types are ultimately from -sqlalchemy.types, and INET is specific to the Postgresql dialect. - -Some dialect level types have the same name as the SQL standard type, -but also provide additional arguments. For example, MySQL implements -the full range of character and string types including additional arguments -such as `collation` and `charset`:: - - from sqlalchemy.dialects.mysql import VARCHAR, TEXT - - table = Table('foo', meta, - Column('col1', VARCHAR(200, collation='binary')), - Column('col2', TEXT(charset='latin1')) - ) - -.. _types_custom: - -Custom Types ------------- - -A variety of methods exist to redefine the behavior of existing types -as well as to provide new ones. - -Overriding Type Compilation -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A frequent need is to force the "string" version of a type, that is -the one rendered in a CREATE TABLE statement or other SQL function -like CAST, to be changed. For example, an application may want -to force the rendering of ``BINARY`` for all platforms -except for one, in which is wants ``BLOB`` to be rendered. Usage -of an existing generic type, in this case :class:`.LargeBinary`, is -preferred for most use cases. But to control -types more accurately, a compilation directive that is per-dialect -can be associated with any type:: - - from sqlalchemy.ext.compiler import compiles - from sqlalchemy.types import BINARY - - @compiles(BINARY, "sqlite") - def compile_binary_sqlite(type_, compiler, **kw): - return "BLOB" - -The above code allows the usage of :class:`.types.BINARY`, which -will produce the string ``BINARY`` against all backends except SQLite, -in which case it will produce ``BLOB``. - -See the section :ref:`type_compilation_extension`, a subsection of -:ref:`sqlalchemy.ext.compiler_toplevel`, for additional examples. - -.. _types_typedecorator: - -Augmenting Existing Types -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`.TypeDecorator` allows the creation of custom types which -add bind-parameter and result-processing behavior to an existing -type object. It is used when additional in-Python marshaling of data -to and from the database is required. - -.. note:: - - The bind- and result-processing of :class:`.TypeDecorator` - is *in addition* to the processing already performed by the hosted - type, which is customized by SQLAlchemy on a per-DBAPI basis to perform - processing specific to that DBAPI. To change the DBAPI-level processing - for an existing type, see the section :ref:`replacing_processors`. - -.. autoclass:: TypeDecorator - :members: - :inherited-members: - - -TypeDecorator Recipes -~~~~~~~~~~~~~~~~~~~~~ -A few key :class:`.TypeDecorator` recipes follow. - -.. _coerce_to_unicode: - -Coercing Encoded Strings to Unicode -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A common source of confusion regarding the :class:`.Unicode` type -is that it is intended to deal *only* with Python ``unicode`` objects -on the Python side, meaning values passed to it as bind parameters -must be of the form ``u'some string'`` if using Python 2 and not 3. -The encoding/decoding functions it performs are only to suit what the -DBAPI in use requires, and are primarily a private implementation detail. - -The use case of a type that can safely receive Python bytestrings, -that is strings that contain non-ASCII characters and are not ``u''`` -objects in Python 2, can be achieved using a :class:`.TypeDecorator` -which coerces as needed:: - - from sqlalchemy.types import TypeDecorator, Unicode - - class CoerceUTF8(TypeDecorator): - """Safely coerce Python bytestrings to Unicode - before passing off to the database.""" - - impl = Unicode - - def process_bind_param(self, value, dialect): - if isinstance(value, str): - value = value.decode('utf-8') - return value - -Rounding Numerics -^^^^^^^^^^^^^^^^^ - -Some database connectors like those of SQL Server choke if a Decimal is passed with too -many decimal places. Here's a recipe that rounds them down:: - - from sqlalchemy.types import TypeDecorator, Numeric - from decimal import Decimal - - class SafeNumeric(TypeDecorator): - """Adds quantization to Numeric.""" - - impl = Numeric - - def __init__(self, *arg, **kw): - TypeDecorator.__init__(self, *arg, **kw) - self.quantize_int = -(self.impl.precision - self.impl.scale) - self.quantize = Decimal(10) ** self.quantize_int - - def process_bind_param(self, value, dialect): - if isinstance(value, Decimal) and \ - value.as_tuple()[2] < self.quantize_int: - value = value.quantize(self.quantize) - return value - -.. _custom_guid_type: - -Backend-agnostic GUID Type -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Receives and returns Python uuid() objects. Uses the PG UUID type -when using Postgresql, CHAR(32) on other backends, storing them -in stringified hex format. Can be modified to store -binary in CHAR(16) if desired:: - - from sqlalchemy.types import TypeDecorator, CHAR - from sqlalchemy.dialects.postgresql import UUID - import uuid - - class GUID(TypeDecorator): - """Platform-independent GUID type. - - Uses Postgresql's UUID type, otherwise uses - CHAR(32), storing as stringified hex values. - - """ - impl = CHAR - - def load_dialect_impl(self, dialect): - if dialect.name == 'postgresql': - return dialect.type_descriptor(UUID()) - else: - return dialect.type_descriptor(CHAR(32)) - - def process_bind_param(self, value, dialect): - if value is None: - return value - elif dialect.name == 'postgresql': - return str(value) - else: - if not isinstance(value, uuid.UUID): - return "%.32x" % uuid.UUID(value) - else: - # hexstring - return "%.32x" % value - - def process_result_value(self, value, dialect): - if value is None: - return value - else: - return uuid.UUID(value) - -Marshal JSON Strings -^^^^^^^^^^^^^^^^^^^^^ - -This type uses ``simplejson`` to marshal Python data structures -to/from JSON. Can be modified to use Python's builtin json encoder:: - - from sqlalchemy.types import TypeDecorator, VARCHAR - import json - - class JSONEncodedDict(TypeDecorator): - """Represents an immutable structure as a json-encoded string. - - Usage:: - - JSONEncodedDict(255) - - """ - - impl = VARCHAR - - def process_bind_param(self, value, dialect): - if value is not None: - value = json.dumps(value) - - return value - - def process_result_value(self, value, dialect): - if value is not None: - value = json.loads(value) - return value - -Note that the ORM by default will not detect "mutability" on such a type - -meaning, in-place changes to values will not be detected and will not be -flushed. Without further steps, you instead would need to replace the existing -value with a new one on each parent object to detect changes. Note that -there's nothing wrong with this, as many applications may not require that the -values are ever mutated once created. For those which do have this requirement, -support for mutability is best applied using the ``sqlalchemy.ext.mutable`` -extension - see the example in :ref:`mutable_toplevel`. - -.. _replacing_processors: - -Replacing the Bind/Result Processing of Existing Types -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Most augmentation of type behavior at the bind/result level -is achieved using :class:`.TypeDecorator`. For the rare scenario -where the specific processing applied by SQLAlchemy at the DBAPI -level needs to be replaced, the SQLAlchemy type can be subclassed -directly, and the ``bind_processor()`` or ``result_processor()`` -methods can be overridden. Doing so requires that the -``adapt()`` method also be overridden. This method is the mechanism -by which SQLAlchemy produces DBAPI-specific type behavior during -statement execution. Overriding it allows a copy of the custom -type to be used in lieu of a DBAPI-specific type. Below we subclass -the :class:`.types.TIME` type to have custom result processing behavior. -The ``process()`` function will receive ``value`` from the DBAPI -cursor directly:: - - class MySpecialTime(TIME): - def __init__(self, special_argument): - super(MySpecialTime, self).__init__() - self.special_argument = special_argument - - def result_processor(self, dialect, coltype): - import datetime - time = datetime.time - def process(value): - if value is not None: - microseconds = value.microseconds - seconds = value.seconds - minutes = seconds / 60 - return time( - minutes / 60, - minutes % 60, - seconds - minutes * 60, - microseconds) - else: - return None - return process - - def adapt(self, impltype): - return MySpecialTime(self.special_argument) - -.. _types_sql_value_processing: - -Applying SQL-level Bind/Result Processing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As seen in the sections :ref:`types_typedecorator` and :ref:`replacing_processors`, -SQLAlchemy allows Python functions to be invoked both when parameters are sent -to a statement, as well as when result rows are loaded from the database, to apply -transformations to the values as they are sent to or from the database. It is also -possible to define SQL-level transformations as well. The rationale here is when -only the relational database contains a particular series of functions that are necessary -to coerce incoming and outgoing data between an application and persistence format. -Examples include using database-defined encryption/decryption functions, as well -as stored procedures that handle geographic data. The Postgis extension to Postgresql -includes an extensive array of SQL functions that are necessary for coercing -data into particular formats. - -Any :class:`.TypeEngine`, :class:`.UserDefinedType` or :class:`.TypeDecorator` subclass -can include implementations of -:meth:`.TypeEngine.bind_expression` and/or :meth:`.TypeEngine.column_expression`, which -when defined to return a non-``None`` value should return a :class:`.ColumnElement` -expression to be injected into the SQL statement, either surrounding -bound parameters or a column expression. For example, to build a ``Geometry`` -type which will apply the Postgis function ``ST_GeomFromText`` to all outgoing -values and the function ``ST_AsText`` to all incoming data, we can create -our own subclass of :class:`.UserDefinedType` which provides these methods -in conjunction with :data:`~.sqlalchemy.sql.expression.func`:: - - from sqlalchemy import func - from sqlalchemy.types import UserDefinedType - - class Geometry(UserDefinedType): - def get_col_spec(self): - return "GEOMETRY" - - def bind_expression(self, bindvalue): - return func.ST_GeomFromText(bindvalue, type_=self) - - def column_expression(self, col): - return func.ST_AsText(col, type_=self) - -We can apply the ``Geometry`` type into :class:`.Table` metadata -and use it in a :func:`.select` construct:: - - geometry = Table('geometry', metadata, - Column('geom_id', Integer, primary_key=True), - Column('geom_data', Geometry) - ) - - print select([geometry]).where( - geometry.c.geom_data == 'LINESTRING(189412 252431,189631 259122)') - -The resulting SQL embeds both functions as appropriate. ``ST_AsText`` -is applied to the columns clause so that the return value is run through -the function before passing into a result set, and ``ST_GeomFromText`` -is run on the bound parameter so that the passed-in value is converted:: - - SELECT geometry.geom_id, ST_AsText(geometry.geom_data) AS geom_data_1 - FROM geometry - WHERE geometry.geom_data = ST_GeomFromText(:geom_data_2) - -The :meth:`.TypeEngine.column_expression` method interacts with the -mechanics of the compiler such that the SQL expression does not interfere -with the labeling of the wrapped expression. Such as, if we rendered -a :func:`.select` against a :func:`.label` of our expression, the string -label is moved to the outside of the wrapped expression:: - - print select([geometry.c.geom_data.label('my_data')]) - -Output:: - - SELECT ST_AsText(geometry.geom_data) AS my_data - FROM geometry - -For an example of subclassing a built in type directly, we subclass -:class:`.postgresql.BYTEA` to provide a ``PGPString``, which will make use of the -Postgresql ``pgcrypto`` extension to encrpyt/decrypt values -transparently:: - - from sqlalchemy import create_engine, String, select, func, \ - MetaData, Table, Column, type_coerce - - from sqlalchemy.dialects.postgresql import BYTEA - - class PGPString(BYTEA): - def __init__(self, passphrase, length=None): - super(PGPString, self).__init__(length) - self.passphrase = passphrase - - def bind_expression(self, bindvalue): - # convert the bind's type from PGPString to - # String, so that it's passed to psycopg2 as is without - # a dbapi.Binary wrapper - bindvalue = type_coerce(bindvalue, String) - return func.pgp_sym_encrypt(bindvalue, self.passphrase) - - def column_expression(self, col): - return func.pgp_sym_decrypt(col, self.passphrase) - - metadata = MetaData() - message = Table('message', metadata, - Column('username', String(50)), - Column('message', - PGPString("this is my passphrase", length=1000)), - ) - - engine = create_engine("postgresql://scott:tiger@localhost/test", echo=True) - with engine.begin() as conn: - metadata.create_all(conn) - - conn.execute(message.insert(), username="some user", - message="this is my message") - - print conn.scalar( - select([message.c.message]).\ - where(message.c.username == "some user") - ) - -The ``pgp_sym_encrypt`` and ``pgp_sym_decrypt`` functions are applied -to the INSERT and SELECT statements:: - - INSERT INTO message (username, message) - VALUES (%(username)s, pgp_sym_encrypt(%(message)s, %(pgp_sym_encrypt_1)s)) - {'username': 'some user', 'message': 'this is my message', - 'pgp_sym_encrypt_1': 'this is my passphrase'} - - SELECT pgp_sym_decrypt(message.message, %(pgp_sym_decrypt_1)s) AS message_1 - FROM message - WHERE message.username = %(username_1)s - {'pgp_sym_decrypt_1': 'this is my passphrase', 'username_1': 'some user'} - - -.. versionadded:: 0.8 Added the :meth:`.TypeEngine.bind_expression` and - :meth:`.TypeEngine.column_expression` methods. - -See also: - -:ref:`examples_postgis` - -.. _types_operators: - -Redefining and Creating New Operators -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -SQLAlchemy Core defines a fixed set of expression operators available to all column expressions. -Some of these operations have the effect of overloading Python's built in operators; -examples of such operators include -:meth:`.ColumnOperators.__eq__` (``table.c.somecolumn == 'foo'``), -:meth:`.ColumnOperators.__invert__` (``~table.c.flag``), -and :meth:`.ColumnOperators.__add__` (``table.c.x + table.c.y``). Other operators are exposed as -explicit methods on column expressions, such as -:meth:`.ColumnOperators.in_` (``table.c.value.in_(['x', 'y'])``) and :meth:`.ColumnOperators.like` -(``table.c.value.like('%ed%')``). - -The Core expression constructs in all cases consult the type of the expression in order to determine -the behavior of existing operators, as well as to locate additional operators that aren't part of -the built in set. The :class:`.TypeEngine` base class defines a root "comparison" implementation -:class:`.TypeEngine.Comparator`, and many specific types provide their own sub-implementations of this -class. User-defined :class:`.TypeEngine.Comparator` implementations can be built directly into a -simple subclass of a particular type in order to override or define new operations. Below, -we create a :class:`.Integer` subclass which overrides the :meth:`.ColumnOperators.__add__` operator:: - - from sqlalchemy import Integer - - class MyInt(Integer): - class comparator_factory(Integer.Comparator): - def __add__(self, other): - return self.op("goofy")(other) - -The above configuration creates a new class ``MyInt``, which -establishes the :attr:`.TypeEngine.comparator_factory` attribute as -referring to a new class, subclassing the :class:`.TypeEngine.Comparator` class -associated with the :class:`.Integer` type. - -Usage:: - - >>> sometable = Table("sometable", metadata, Column("data", MyInt)) - >>> print sometable.c.data + 5 - sometable.data goofy :data_1 - -The implementation for :meth:`.ColumnOperators.__add__` is consulted -by an owning SQL expression, by instantiating the :class:`.TypeEngine.Comparator` with -itself as the ``expr`` attribute. The mechanics of the expression -system are such that operations continue recursively until an -expression object produces a new SQL expression construct. Above, we -could just as well have said ``self.expr.op("goofy")(other)`` instead -of ``self.op("goofy")(other)``. - -New methods added to a :class:`.TypeEngine.Comparator` are exposed on an -owning SQL expression -using a ``__getattr__`` scheme, which exposes methods added to -:class:`.TypeEngine.Comparator` onto the owning :class:`.ColumnElement`. -For example, to add a ``log()`` function -to integers:: - - from sqlalchemy import Integer, func - - class MyInt(Integer): - class comparator_factory(Integer.Comparator): - def log(self, other): - return func.log(self.expr, other) - -Using the above type:: - - >>> print sometable.c.data.log(5) - log(:log_1, :log_2) - - -Unary operations -are also possible. For example, to add an implementation of the -Postgresql factorial operator, we combine the :class:`.UnaryExpression` construct -along with a :class:`.custom_op` to produce the factorial expression:: - - from sqlalchemy import Integer - from sqlalchemy.sql.expression import UnaryExpression - from sqlalchemy.sql import operators - - class MyInteger(Integer): - class comparator_factory(Integer.Comparator): - def factorial(self): - return UnaryExpression(self.expr, - modifier=operators.custom_op("!"), - type_=MyInteger) - -Using the above type:: - - >>> from sqlalchemy.sql import column - >>> print column('x', MyInteger).factorial() - x ! - -See also: - -:attr:`.TypeEngine.comparator_factory` - -.. versionadded:: 0.8 The expression system was enhanced to support - customization of operators on a per-type level. - - -Creating New Types -~~~~~~~~~~~~~~~~~~ - -The :class:`.UserDefinedType` class is provided as a simple base class -for defining entirely new database types. Use this to represent native -database types not known by SQLAlchemy. If only Python translation behavior -is needed, use :class:`.TypeDecorator` instead. - -.. autoclass:: UserDefinedType - :members: - - -.. _types_api: - -Base Type API --------------- - -.. autoclass:: TypeEngine - :members: - - -.. autoclass:: Concatenable - :members: - :inherited-members: - - -.. autoclass:: NullType - - -.. autoclass:: Variant - - :members: with_variant, __init__ + type_basics + custom_types + type_api \ No newline at end of file diff --git a/doc/build/faq.rst b/doc/build/faq.rst deleted file mode 100644 index 8c3bd24f4..000000000 --- a/doc/build/faq.rst +++ /dev/null @@ -1,1504 +0,0 @@ -:orphan: - -.. _faq_toplevel: - -============================ -Frequently Asked Questions -============================ - -.. contents:: - :local: - :class: faq - :backlinks: none - - -Connections / Engines -===================== - -How do I configure logging? ---------------------------- - -See :ref:`dbengine_logging`. - -How do I pool database connections? Are my connections pooled? ----------------------------------------------------------------- - -SQLAlchemy performs application-level connection pooling automatically -in most cases. With the exception of SQLite, a :class:`.Engine` object -refers to a :class:`.QueuePool` as a source of connectivity. - -For more detail, see :ref:`engines_toplevel` and :ref:`pooling_toplevel`. - -How do I pass custom connect arguments to my database API? ------------------------------------------------------------ - -The :func:`.create_engine` call accepts additional arguments either -directly via the ``connect_args`` keyword argument:: - - e = create_engine("mysql://scott:tiger@localhost/test", - connect_args={"encoding": "utf8"}) - -Or for basic string and integer arguments, they can usually be specified -in the query string of the URL:: - - e = create_engine("mysql://scott:tiger@localhost/test?encoding=utf8") - -.. seealso:: - - :ref:`custom_dbapi_args` - -"MySQL Server has gone away" ----------------------------- - -There are two major causes for this error: - -1. The MySQL client closes connections which have been idle for a set period -of time, defaulting to eight hours. This can be avoided by using the ``pool_recycle`` -setting with :func:`.create_engine`, described at :ref:`mysql_connection_timeouts`. - -2. Usage of the MySQLdb :term:`DBAPI`, or a similar DBAPI, in a non-threadsafe manner, or in an otherwise -inappropriate way. The MySQLdb connection object is not threadsafe - this expands -out to any SQLAlchemy system that links to a single connection, which includes the ORM -:class:`.Session`. For background -on how :class:`.Session` should be used in a multithreaded environment, -see :ref:`session_faq_threadsafe`. - -Why does SQLAlchemy issue so many ROLLBACKs? ---------------------------------------------- - -SQLAlchemy currently assumes DBAPI connections are in "non-autocommit" mode - -this is the default behavior of the Python database API, meaning it -must be assumed that a transaction is always in progress. The -connection pool issues ``connection.rollback()`` when a connection is returned. -This is so that any transactional resources remaining on the connection are -released. On a database like Postgresql or MSSQL where table resources are -aggressively locked, this is critical so that rows and tables don't remain -locked within connections that are no longer in use. An application can -otherwise hang. It's not just for locks, however, and is equally critical on -any database that has any kind of transaction isolation, including MySQL with -InnoDB. Any connection that is still inside an old transaction will return -stale data, if that data was already queried on that connection within -isolation. For background on why you might see stale data even on MySQL, see -http://dev.mysql.com/doc/refman/5.1/en/innodb-transaction-model.html - -I'm on MyISAM - how do I turn it off? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The behavior of the connection pool's connection return behavior can be -configured using ``reset_on_return``:: - - from sqlalchemy import create_engine - from sqlalchemy.pool import QueuePool - - engine = create_engine('mysql://scott:tiger@localhost/myisam_database', pool=QueuePool(reset_on_return=False)) - -I'm on SQL Server - how do I turn those ROLLBACKs into COMMITs? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``reset_on_return`` accepts the values ``commit``, ``rollback`` in addition -to ``True``, ``False``, and ``None``. Setting to ``commit`` will cause -a COMMIT as any connection is returned to the pool:: - - engine = create_engine('mssql://scott:tiger@mydsn', pool=QueuePool(reset_on_return='commit')) - - -I am using multiple connections with a SQLite database (typically to test transaction operation), and my test program is not working! ----------------------------------------------------------------------------------------------------------------------------------------------------------- - -If using a SQLite ``:memory:`` database, or a version of SQLAlchemy prior -to version 0.7, the default connection pool is the :class:`.SingletonThreadPool`, -which maintains exactly one SQLite connection per thread. So two -connections in use in the same thread will actually be the same SQLite -connection. Make sure you're not using a :memory: database and -use :class:`.NullPool`, which is the default for non-memory databases in -current SQLAlchemy versions. - -.. seealso:: - - :ref:`pysqlite_threading_pooling` - info on PySQLite's behavior. - -How do I get at the raw DBAPI connection when using an Engine? --------------------------------------------------------------- - -With a regular SA engine-level Connection, you can get at a pool-proxied -version of the DBAPI connection via the :attr:`.Connection.connection` attribute on -:class:`.Connection`, and for the really-real DBAPI connection you can call the -:attr:`.ConnectionFairy.connection` attribute on that - but there should never be any need to access -the non-pool-proxied DBAPI connection, as all methods are proxied through:: - - engine = create_engine(...) - conn = engine.connect() - conn.connection. - cursor = conn.connection.cursor() - -You must ensure that you revert any isolation level settings or other -operation-specific settings on the connection back to normal before returning -it to the pool. - -As an alternative to reverting settings, you can call the :meth:`.Connection.detach` method on -either :class:`.Connection` or the proxied connection, which will de-associate -the connection from the pool such that it will be closed and discarded -when :meth:`.Connection.close` is called:: - - conn = engine.connect() - conn.detach() # detaches the DBAPI connection from the connection pool - conn.connection. - conn.close() # connection is closed for real, the pool replaces it with a new connection - -MetaData / Schema -================== - -My program is hanging when I say ``table.drop()`` / ``metadata.drop_all()`` ----------------------------------------------------------------------------- - -This usually corresponds to two conditions: 1. using PostgreSQL, which is really -strict about table locks, and 2. you have a connection still open which -contains locks on the table and is distinct from the connection being used for -the DROP statement. Heres the most minimal version of the pattern:: - - connection = engine.connect() - result = connection.execute(mytable.select()) - - mytable.drop(engine) - -Above, a connection pool connection is still checked out; furthermore, the -result object above also maintains a link to this connection. If -"implicit execution" is used, the result will hold this connection opened until -the result object is closed or all rows are exhausted. - -The call to ``mytable.drop(engine)`` attempts to emit DROP TABLE on a second -connection procured from the :class:`.Engine` which will lock. - -The solution is to close out all connections before emitting DROP TABLE:: - - connection = engine.connect() - result = connection.execute(mytable.select()) - - # fully read result sets - result.fetchall() - - # close connections - connection.close() - - # now locks are removed - mytable.drop(engine) - -Does SQLAlchemy support ALTER TABLE, CREATE VIEW, CREATE TRIGGER, Schema Upgrade Functionality? ------------------------------------------------------------------------------------------------ - -General ALTER support isn't present in SQLAlchemy directly. For special DDL -on an ad-hoc basis, the :class:`.DDL` and related constructs can be used. -See :doc:`core/ddl` for a discussion on this subject. - -A more comprehensive option is to use schema migration tools, such as Alembic -or SQLAlchemy-Migrate; see :ref:`schema_migrations` for discussion on this. - -How can I sort Table objects in order of their dependency? ------------------------------------------------------------ - -This is available via the :attr:`.MetaData.sorted_tables` function:: - - metadata = MetaData() - # ... add Table objects to metadata - ti = metadata.sorted_tables: - for t in ti: - print t - -How can I get the CREATE TABLE/ DROP TABLE output as a string? ---------------------------------------------------------------- - -Modern SQLAlchemy has clause constructs which represent DDL operations. These -can be rendered to strings like any other SQL expression:: - - from sqlalchemy.schema import CreateTable - - print CreateTable(mytable) - -To get the string specific to a certain engine:: - - print CreateTable(mytable).compile(engine) - -There's also a special form of :class:`.Engine` that can let you dump an entire -metadata creation sequence, using this recipe:: - - def dump(sql, *multiparams, **params): - print sql.compile(dialect=engine.dialect) - engine = create_engine('postgresql://', strategy='mock', executor=dump) - metadata.create_all(engine, checkfirst=False) - -The `Alembic `_ tool also supports -an "offline" SQL generation mode that renders database migrations as SQL scripts. - -How can I subclass Table/Column to provide certain behaviors/configurations? ------------------------------------------------------------------------------- - -:class:`.Table` and :class:`.Column` are not good targets for direct subclassing. -However, there are simple ways to get on-construction behaviors using creation -functions, and behaviors related to the linkages between schema objects such as -constraint conventions or naming conventions using attachment events. -An example of many of these -techniques can be seen at `Naming Conventions `_. - - -SQL Expressions -================= - -.. _faq_sql_expression_string: - -How do I render SQL expressions as strings, possibly with bound parameters inlined? ------------------------------------------------------------------------------------- - -The "stringification" of a SQLAlchemy statement or Query in the vast majority -of cases is as simple as:: - - print(str(statement)) - -this applies both to an ORM :class:`~.orm.query.Query` as well as any :func:`.select` or other -statement. Additionally, to get the statement as compiled to a -specific dialect or engine, if the statement itself is not already -bound to one you can pass this in to :meth:`.ClauseElement.compile`:: - - print(statement.compile(someengine)) - -or without an :class:`.Engine`:: - - from sqlalchemy.dialects import postgresql - print(statement.compile(dialect=postgresql.dialect())) - -When given an ORM :class:`~.orm.query.Query` object, in order to get at the -:meth:`.ClauseElement.compile` -method we only need access the :attr:`~.orm.query.Query.statement` -accessor first:: - - statement = query.statement - print(statement.compile(someengine)) - -The above forms will render the SQL statement as it is passed to the Python -:term:`DBAPI`, which includes that bound parameters are not rendered inline. -SQLAlchemy normally does not stringify bound parameters, as this is handled -appropriately by the Python DBAPI, not to mention bypassing bound -parameters is probably the most widely exploited security hole in -modern web applications. SQLAlchemy has limited ability to do this -stringification in certain circumstances such as that of emitting DDL. -In order to access this functionality one can use the ``literal_binds`` -flag, passed to ``compile_kwargs``:: - - from sqlalchemy.sql import table, column, select - - t = table('t', column('x')) - - s = select([t]).where(t.c.x == 5) - - print(s.compile(compile_kwargs={"literal_binds": True})) - -the above approach has the caveats that it is only supported for basic -types, such as ints and strings, and furthermore if a :func:`.bindparam` -without a pre-set value is used directly, it won't be able to -stringify that either. - -To support inline literal rendering for types not supported, implement -a :class:`.TypeDecorator` for the target type which includes a -:meth:`.TypeDecorator.process_literal_param` method:: - - from sqlalchemy import TypeDecorator, Integer - - - class MyFancyType(TypeDecorator): - impl = Integer - - def process_literal_param(self, value, dialect): - return "my_fancy_formatting(%s)" % value - - from sqlalchemy import Table, Column, MetaData - - tab = Table('mytable', MetaData(), Column('x', MyFancyType())) - - print( - tab.select().where(tab.c.x > 5).compile( - compile_kwargs={"literal_binds": True}) - ) - -producing output like:: - - SELECT mytable.x - FROM mytable - WHERE mytable.x > my_fancy_formatting(5) - - -Why does ``.col.in_([])`` Produce ``col != col``? Why not ``1=0``? -------------------------------------------------------------------- - -A little introduction to the issue. The IN operator in SQL, given a list of -elements to compare against a column, generally does not accept an empty list, -that is while it is valid to say:: - - column IN (1, 2, 3) - -it's not valid to say:: - - column IN () - -SQLAlchemy's :meth:`.Operators.in_` operator, when given an empty list, produces this -expression:: - - column != column - -As of version 0.6, it also produces a warning stating that a less efficient -comparison operation will be rendered. This expression is the only one that is -both database agnostic and produces correct results. - -For example, the naive approach of "just evaluate to false, by comparing 1=0 -or 1!=1", does not handle nulls properly. An expression like:: - - NOT column != column - -will not return a row when "column" is null, but an expression which does not -take the column into account:: - - NOT 1=0 - -will. - -Closer to the mark is the following CASE expression:: - - CASE WHEN column IS NOT NULL THEN 1=0 ELSE NULL END - -We don't use this expression due to its verbosity, and its also not -typically accepted by Oracle within a WHERE clause - depending -on how you phrase it, you'll either get "ORA-00905: missing keyword" or -"ORA-00920: invalid relational operator". It's also still less efficient than -just rendering SQL without the clause altogether (or not issuing the SQL at -all, if the statement is just a simple search). - -The best approach therefore is to avoid the usage of IN given an argument list -of zero length. Instead, don't emit the Query in the first place, if no rows -should be returned. The warning is best promoted to a full error condition -using the Python warnings filter (see http://docs.python.org/library/warnings.html). - -ORM Configuration -================== - -.. _faq_mapper_primary_key: - -How do I map a table that has no primary key? ---------------------------------------------- - -The SQLAlchemy ORM, in order to map to a particular table, needs there to be -at least one column denoted as a primary key column; multiple-column, -i.e. composite, primary keys are of course entirely feasible as well. These -columns do **not** need to be actually known to the database as primary key -columns, though it's a good idea that they are. It's only necessary that the columns -*behave* as a primary key does, e.g. as a unique and not nullable identifier -for a row. - -Most ORMs require that objects have some kind of primary key defined -because the object in memory must correspond to a uniquely identifiable -row in the database table; at the very least, this allows the -object can be targeted for UPDATE and DELETE statements which will affect only -that object's row and no other. However, the importance of the primary key -goes far beyond that. In SQLAlchemy, all ORM-mapped objects are at all times -linked uniquely within a :class:`.Session` -to their specific database row using a pattern called the :term:`identity map`, -a pattern that's central to the unit of work system employed by SQLAlchemy, -and is also key to the most common (and not-so-common) patterns of ORM usage. - - -.. note:: - - It's important to note that we're only talking about the SQLAlchemy ORM; an - application which builds on Core and deals only with :class:`.Table` objects, - :func:`.select` constructs and the like, **does not** need any primary key - to be present on or associated with a table in any way (though again, in SQL, all tables - should really have some kind of primary key, lest you need to actually - update or delete specific rows). - -In almost all cases, a table does have a so-called :term:`candidate key`, which is a column or series -of columns that uniquely identify a row. If a table truly doesn't have this, and has actual -fully duplicate rows, the table is not corresponding to `first normal form `_ and cannot be mapped. Otherwise, whatever columns comprise the best candidate key can be -applied directly to the mapper:: - - class SomeClass(Base): - __table__ = some_table_with_no_pk - __mapper_args__ = { - 'primary_key':[some_table_with_no_pk.c.uid, some_table_with_no_pk.c.bar] - } - -Better yet is when using fully declared table metadata, use the ``primary_key=True`` -flag on those columns:: - - class SomeClass(Base): - __tablename__ = "some_table_with_no_pk" - - uid = Column(Integer, primary_key=True) - bar = Column(String, primary_key=True) - -All tables in a relational database should have primary keys. Even a many-to-many -association table - the primary key would be the composite of the two association -columns:: - - CREATE TABLE my_association ( - user_id INTEGER REFERENCES user(id), - account_id INTEGER REFERENCES account(id), - PRIMARY KEY (user_id, account_id) - ) - - -How do I configure a Column that is a Python reserved word or similar? ----------------------------------------------------------------------------- - -Column-based attributes can be given any name desired in the mapping. See -:ref:`mapper_column_distinct_names`. - -How do I get a list of all columns, relationships, mapped attributes, etc. given a mapped class? -------------------------------------------------------------------------------------------------- - -This information is all available from the :class:`.Mapper` object. - -To get at the :class:`.Mapper` for a particular mapped class, call the -:func:`.inspect` function on it:: - - from sqlalchemy import inspect - - mapper = inspect(MyClass) - -From there, all information about the class can be acquired using such methods as: - -* :attr:`.Mapper.attrs` - a namespace of all mapped attributes. The attributes - themselves are instances of :class:`.MapperProperty`, which contain additional - attributes that can lead to the mapped SQL expression or column, if applicable. - -* :attr:`.Mapper.column_attrs` - the mapped attribute namespace - limited to column and SQL expression attributes. You might want to use - :attr:`.Mapper.columns` to get at the :class:`.Column` objects directly. - -* :attr:`.Mapper.relationships` - namespace of all :class:`.RelationshipProperty` attributes. - -* :attr:`.Mapper.all_orm_descriptors` - namespace of all mapped attributes, plus user-defined - attributes defined using systems such as :class:`.hybrid_property`, :class:`.AssociationProxy` and others. - -* :attr:`.Mapper.columns` - A namespace of :class:`.Column` objects and other named - SQL expressions associated with the mapping. - -* :attr:`.Mapper.mapped_table` - The :class:`.Table` or other selectable to which - this mapper is mapped. - -* :attr:`.Mapper.local_table` - The :class:`.Table` that is "local" to this mapper; - this differs from :attr:`.Mapper.mapped_table` in the case of a mapper mapped - using inheritance to a composed selectable. - -.. _faq_combining_columns: - -I'm getting a warning or error about "Implicitly combining column X under attribute Y" --------------------------------------------------------------------------------------- - -This condition refers to when a mapping contains two columns that are being -mapped under the same attribute name due to their name, but there's no indication -that this is intentional. A mapped class needs to have explicit names for -every attribute that is to store an independent value; when two columns have the -same name and aren't disambiguated, they fall under the same attribute and -the effect is that the value from one column is **copied** into the other, based -on which column was assigned to the attribute first. - -This behavior is often desirable and is allowed without warning in the case -where the two columns are linked together via a foreign key relationship -within an inheritance mapping. When the warning or exception occurs, the -issue can be resolved by either assigning the columns to differently-named -attributes, or if combining them together is desired, by using -:func:`.column_property` to make this explicit. - -Given the example as follows:: - - from sqlalchemy import Integer, Column, ForeignKey - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class A(Base): - __tablename__ = 'a' - - id = Column(Integer, primary_key=True) - - class B(A): - __tablename__ = 'b' - - id = Column(Integer, primary_key=True) - a_id = Column(Integer, ForeignKey('a.id')) - -As of SQLAlchemy version 0.9.5, the above condition is detected, and will -warn that the ``id`` column of ``A`` and ``B`` is being combined under -the same-named attribute ``id``, which above is a serious issue since it means -that a ``B`` object's primary key will always mirror that of its ``A``. - -A mapping which resolves this is as follows:: - - class A(Base): - __tablename__ = 'a' - - id = Column(Integer, primary_key=True) - - class B(A): - __tablename__ = 'b' - - b_id = Column('id', Integer, primary_key=True) - a_id = Column(Integer, ForeignKey('a.id')) - -Suppose we did want ``A.id`` and ``B.id`` to be mirrors of each other, despite -the fact that ``B.a_id`` is where ``A.id`` is related. We could combine -them together using :func:`.column_property`:: - - class A(Base): - __tablename__ = 'a' - - id = Column(Integer, primary_key=True) - - class B(A): - __tablename__ = 'b' - - # probably not what you want, but this is a demonstration - id = column_property(Column(Integer, primary_key=True), A.id) - a_id = Column(Integer, ForeignKey('a.id')) - - - -I'm using Declarative and setting primaryjoin/secondaryjoin using an ``and_()`` or ``or_()``, and I am getting an error message about foreign keys. ------------------------------------------------------------------------------------------------------------------------------------------------------------------- - -Are you doing this?:: - - class MyClass(Base): - # .... - - foo = relationship("Dest", primaryjoin=and_("MyClass.id==Dest.foo_id", "MyClass.foo==Dest.bar")) - -That's an ``and_()`` of two string expressions, which SQLAlchemy cannot apply any mapping towards. Declarative allows :func:`.relationship` arguments to be specified as strings, which are converted into expression objects using ``eval()``. But this doesn't occur inside of an ``and_()`` expression - it's a special operation declarative applies only to the *entirety* of what's passed to primaryjoin or other arguments as a string:: - - class MyClass(Base): - # .... - - foo = relationship("Dest", primaryjoin="and_(MyClass.id==Dest.foo_id, MyClass.foo==Dest.bar)") - -Or if the objects you need are already available, skip the strings:: - - class MyClass(Base): - # .... - - foo = relationship(Dest, primaryjoin=and_(MyClass.id==Dest.foo_id, MyClass.foo==Dest.bar)) - -The same idea applies to all the other arguments, such as ``foreign_keys``:: - - # wrong ! - foo = relationship(Dest, foreign_keys=["Dest.foo_id", "Dest.bar_id"]) - - # correct ! - foo = relationship(Dest, foreign_keys="[Dest.foo_id, Dest.bar_id]") - - # also correct ! - foo = relationship(Dest, foreign_keys=[Dest.foo_id, Dest.bar_id]) - - # if you're using columns from the class that you're inside of, just use the column objects ! - class MyClass(Base): - foo_id = Column(...) - bar_id = Column(...) - # ... - - foo = relationship(Dest, foreign_keys=[foo_id, bar_id]) - -.. _faq_subqueryload_limit_sort: - -Why is ``ORDER BY`` required with ``LIMIT`` (especially with ``subqueryload()``)? ---------------------------------------------------------------------------------- - -A relational database can return rows in any -arbitrary order, when an explicit ordering is not set. -While this ordering very often corresponds to the natural -order of rows within a table, this is not the case for all databases and -all queries. The consequence of this is that any query that limits rows -using ``LIMIT`` or ``OFFSET`` should **always** specify an ``ORDER BY``. -Otherwise, it is not deterministic which rows will actually be returned. - -When we use a SQLAlchemy method like :meth:`.Query.first`, we are in fact -applying a ``LIMIT`` of one to the query, so without an explicit ordering -it is not deterministic what row we actually get back. -While we may not notice this for simple queries on databases that usually -returns rows in their natural -order, it becomes much more of an issue if we also use :func:`.orm.subqueryload` -to load related collections, and we may not be loading the collections -as intended. - -SQLAlchemy implements :func:`.orm.subqueryload` by issuing a separate query, -the results of which are matched up to the results from the first query. -We see two queries emitted like this: - -.. sourcecode:: python+sql - - >>> session.query(User).options(subqueryload(User.addresses)).all() - {opensql}-- the "main" query - SELECT users.id AS users_id - FROM users - {stop} - {opensql}-- the "load" query issued by subqueryload - SELECT addresses.id AS addresses_id, - addresses.user_id AS addresses_user_id, - anon_1.users_id AS anon_1_users_id - FROM (SELECT users.id AS users_id FROM users) AS anon_1 - JOIN addresses ON anon_1.users_id = addresses.user_id - ORDER BY anon_1.users_id - -The second query embeds the first query as a source of rows. -When the inner query uses ``OFFSET`` and/or ``LIMIT`` without ordering, -the two queries may not see the same results: - -.. sourcecode:: python+sql - - >>> user = session.query(User).options(subqueryload(User.addresses)).first() - {opensql}-- the "main" query - SELECT users.id AS users_id - FROM users - LIMIT 1 - {stop} - {opensql}-- the "load" query issued by subqueryload - SELECT addresses.id AS addresses_id, - addresses.user_id AS addresses_user_id, - anon_1.users_id AS anon_1_users_id - FROM (SELECT users.id AS users_id FROM users LIMIT 1) AS anon_1 - JOIN addresses ON anon_1.users_id = addresses.user_id - ORDER BY anon_1.users_id - -Depending on database specifics, there is -a chance we may get the a result like the following for the two queries:: - - -- query #1 - +--------+ - |users_id| - +--------+ - | 1| - +--------+ - - -- query #2 - +------------+-----------------+---------------+ - |addresses_id|addresses_user_id|anon_1_users_id| - +------------+-----------------+---------------+ - | 3| 2| 2| - +------------+-----------------+---------------+ - | 4| 2| 2| - +------------+-----------------+---------------+ - -Above, we receive two ``addresses`` rows for ``user.id`` of 2, and none for -1. We've wasted two rows and failed to actually load the collection. This -is an insidious error because without looking at the SQL and the results, the -ORM will not show that there's any issue; if we access the ``addresses`` -for the ``User`` we have, it will emit a lazy load for the collection and we -won't see that anything actually went wrong. - -The solution to this problem is to always specify a deterministic sort order, -so that the main query always returns the same set of rows. This generally -means that you should :meth:`.Query.order_by` on a unique column on the table. -The primary key is a good choice for this:: - - session.query(User).options(subqueryload(User.addresses)).order_by(User.id).first() - -Note that :func:`.joinedload` does not suffer from the same problem because -only one query is ever issued, so the load query cannot be different from the -main query. - -.. seealso:: - - :ref:`subqueryload_ordering` - -.. _faq_performance: - -Performance -=========== - -.. _faq_how_to_profile: - -How can I profile a SQLAlchemy powered application? ---------------------------------------------------- - -Looking for performance issues typically involves two stratgies. One -is query profiling, and the other is code profiling. - -Query Profiling -^^^^^^^^^^^^^^^^ - -Sometimes just plain SQL logging (enabled via python's logging module -or via the ``echo=True`` argument on :func:`.create_engine`) can give an -idea how long things are taking. For example, if you log something -right after a SQL operation, you'd see something like this in your -log:: - - 17:37:48,325 INFO [sqlalchemy.engine.base.Engine.0x...048c] SELECT ... - 17:37:48,326 INFO [sqlalchemy.engine.base.Engine.0x...048c] {} - 17:37:48,660 DEBUG [myapp.somemessage] - -if you logged ``myapp.somemessage`` right after the operation, you know -it took 334ms to complete the SQL part of things. - -Logging SQL will also illustrate if dozens/hundreds of queries are -being issued which could be better organized into much fewer queries. -When using the SQLAlchemy ORM, the "eager loading" -feature is provided to partially (:func:`.contains_eager()`) or fully -(:func:`.joinedload()`, :func:`.subqueryload()`) -automate this activity, but without -the ORM "eager loading" typically means to use joins so that results across multiple -tables can be loaded in one result set instead of multiplying numbers -of queries as more depth is added (i.e. ``r + r*r2 + r*r2*r3`` ...) - -For more long-term profiling of queries, or to implement an application-side -"slow query" monitor, events can be used to intercept cursor executions, -using a recipe like the following:: - - from sqlalchemy import event - from sqlalchemy.engine import Engine - import time - import logging - - logging.basicConfig() - logger = logging.getLogger("myapp.sqltime") - logger.setLevel(logging.DEBUG) - - @event.listens_for(Engine, "before_cursor_execute") - def before_cursor_execute(conn, cursor, statement, - parameters, context, executemany): - conn.info.setdefault('query_start_time', []).append(time.time()) - logger.debug("Start Query: %s", statement) - - @event.listens_for(Engine, "after_cursor_execute") - def after_cursor_execute(conn, cursor, statement, - parameters, context, executemany): - total = time.time() - conn.info['query_start_time'].pop(-1) - logger.debug("Query Complete!") - logger.debug("Total Time: %f", total) - -Above, we use the :meth:`.ConnectionEvents.before_cursor_execute` and -:meth:`.ConnectionEvents.after_cursor_execute` events to establish an interception -point around when a statement is executed. We attach a timer onto the -connection using the :class:`._ConnectionRecord.info` dictionary; we use a -stack here for the occasional case where the cursor execute events may be nested. - -Code Profiling -^^^^^^^^^^^^^^ - -If logging reveals that individual queries are taking too long, you'd -need a breakdown of how much time was spent within the database -processing the query, sending results over the network, being handled -by the :term:`DBAPI`, and finally being received by SQLAlchemy's result set -and/or ORM layer. Each of these stages can present their own -individual bottlenecks, depending on specifics. - -For that you need to use the -`Python Profiling Module `_. -Below is a simple recipe which works profiling into a context manager:: - - import cProfile - import StringIO - import pstats - import contextlib - - @contextlib.contextmanager - def profiled(): - pr = cProfile.Profile() - pr.enable() - yield - pr.disable() - s = StringIO.StringIO() - ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') - ps.print_stats() - # uncomment this to see who's calling what - # ps.print_callers() - print s.getvalue() - -To profile a section of code:: - - with profiled(): - Session.query(FooClass).filter(FooClass.somevalue==8).all() - -The output of profiling can be used to give an idea where time is -being spent. A section of profiling output looks like this:: - - 13726 function calls (13042 primitive calls) in 0.014 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 222/21 0.001 0.000 0.011 0.001 lib/sqlalchemy/orm/loading.py:26(instances) - 220/20 0.002 0.000 0.010 0.001 lib/sqlalchemy/orm/loading.py:327(_instance) - 220/20 0.000 0.000 0.010 0.000 lib/sqlalchemy/orm/loading.py:284(populate_state) - 20 0.000 0.000 0.010 0.000 lib/sqlalchemy/orm/strategies.py:987(load_collection_from_subq) - 20 0.000 0.000 0.009 0.000 lib/sqlalchemy/orm/strategies.py:935(get) - 1 0.000 0.000 0.009 0.009 lib/sqlalchemy/orm/strategies.py:940(_load) - 21 0.000 0.000 0.008 0.000 lib/sqlalchemy/orm/strategies.py:942() - 2 0.000 0.000 0.004 0.002 lib/sqlalchemy/orm/query.py:2400(__iter__) - 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/orm/query.py:2414(_execute_and_instances) - 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/engine/base.py:659(execute) - 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/sql/elements.py:321(_execute_on_connection) - 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/engine/base.py:788(_execute_clauseelement) - - ... - -Above, we can see that the ``instances()`` SQLAlchemy function was called 222 -times (recursively, and 21 times from the outside), taking a total of .011 -seconds for all calls combined. - -Execution Slowness -^^^^^^^^^^^^^^^^^^ - -The specifics of these calls can tell us where the time is being spent. -If for example, you see time being spent within ``cursor.execute()``, -e.g. against the DBAPI:: - - 2 0.102 0.102 0.204 0.102 {method 'execute' of 'sqlite3.Cursor' objects} - -this would indicate that the database is taking a long time to start returning -results, and it means your query should be optimized, either by adding indexes -or restructuring the query and/or underlying schema. For that task, -analysis of the query plan is warranted, using a system such as EXPLAIN, -SHOW PLAN, etc. as is provided by the database backend. - -Result Fetching Slowness - Core -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If on the other hand you see many thousands of calls related to fetching rows, -or very long calls to ``fetchall()``, it may -mean your query is returning more rows than expected, or that the fetching -of rows itself is slow. The ORM itself typically uses ``fetchall()`` to fetch -rows (or ``fetchmany()`` if the :meth:`.Query.yield_per` option is used). - -An inordinately large number of rows would be indicated -by a very slow call to ``fetchall()`` at the DBAPI level:: - - 2 0.300 0.600 0.300 0.600 {method 'fetchall' of 'sqlite3.Cursor' objects} - -An unexpectedly large number of rows, even if the ultimate result doesn't seem -to have many rows, can be the result of a cartesian product - when multiple -sets of rows are combined together without appropriately joining the tables -together. It's often easy to produce this behavior with SQLAlchemy Core or -ORM query if the wrong :class:`.Column` objects are used in a complex query, -pulling in additional FROM clauses that are unexpected. - -On the other hand, a fast call to ``fetchall()`` at the DBAPI level, but then -slowness when SQLAlchemy's :class:`.ResultProxy` is asked to do a ``fetchall()``, -may indicate slowness in processing of datatypes, such as unicode conversions -and similar:: - - # the DBAPI cursor is fast... - 2 0.020 0.040 0.020 0.040 {method 'fetchall' of 'sqlite3.Cursor' objects} - - ... - - # but SQLAlchemy's result proxy is slow, this is type-level processing - 2 0.100 0.200 0.100 0.200 lib/sqlalchemy/engine/result.py:778(fetchall) - -In some cases, a backend might be doing type-level processing that isn't -needed. More specifically, seeing calls within the type API that are slow -are better indicators - below is what it looks like when we use a type like -this:: - - from sqlalchemy import TypeDecorator - import time - - class Foo(TypeDecorator): - impl = String - - def process_result_value(self, value, thing): - # intentionally add slowness for illustration purposes - time.sleep(.001) - return value - -the profiling output of this intentionally slow operation can be seen like this:: - - 200 0.001 0.000 0.237 0.001 lib/sqlalchemy/sql/type_api.py:911(process) - 200 0.001 0.000 0.236 0.001 test.py:28(process_result_value) - 200 0.235 0.001 0.235 0.001 {time.sleep} - -that is, we see many expensive calls within the ``type_api`` system, and the actual -time consuming thing is the ``time.sleep()`` call. - -Make sure to check the :doc:`Dialect documentation ` -for notes on known performance tuning suggestions at this level, especially for -databases like Oracle. There may be systems related to ensuring numeric accuracy -or string processing that may not be needed in all cases. - -There also may be even more low-level points at which row-fetching performance is suffering; -for example, if time spent seems to focus on a call like ``socket.receive()``, -that could indicate that everything is fast except for the actual network connection, -and too much time is spent with data moving over the network. - -Result Fetching Slowness - ORM -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To detect slowness in ORM fetching of rows (which is the most common area -of performance concern), calls like ``populate_state()`` and ``_instance()`` will -illustrate individual ORM object populations:: - - # the ORM calls _instance for each ORM-loaded row it sees, and - # populate_state for each ORM-loaded row that results in the population - # of an object's attributes - 220/20 0.001 0.000 0.010 0.000 lib/sqlalchemy/orm/loading.py:327(_instance) - 220/20 0.000 0.000 0.009 0.000 lib/sqlalchemy/orm/loading.py:284(populate_state) - -The ORM's slowness in turning rows into ORM-mapped objects is a product -of the complexity of this operation combined with the overhead of cPython. -Common strategies to mitigate this include: - -* fetch individual columns instead of full entities, that is:: - - session.query(User.id, User.name) - - instead of:: - - session.query(User) - -* Use :class:`.Bundle` objects to organize column-based results:: - - u_b = Bundle('user', User.id, User.name) - a_b = Bundle('address', Address.id, Address.email) - - for user, address in session.query(u_b, a_b).join(User.addresses): - # ... - -* Use result caching - see :ref:`examples_caching` for an in-depth example - of this. - -* Consider a faster interpreter like that of Pypy. - -The output of a profile can be a little daunting but after some -practice they are very easy to read. - -.. seealso:: - - :ref:`examples_performance` - a suite of performance demonstrations - with bundled profiling capabilities. - -I'm inserting 400,000 rows with the ORM and it's really slow! --------------------------------------------------------------- - -The SQLAlchemy ORM uses the :term:`unit of work` pattern when synchronizing -changes to the database. This pattern goes far beyond simple "inserts" -of data. It includes that attributes which are assigned on objects are -received using an attribute instrumentation system which tracks -changes on objects as they are made, includes that all rows inserted -are tracked in an identity map which has the effect that for each row -SQLAlchemy must retrieve its "last inserted id" if not already given, -and also involves that rows to be inserted are scanned and sorted for -dependencies as needed. Objects are also subject to a fair degree of -bookkeeping in order to keep all of this running, which for a very -large number of rows at once can create an inordinate amount of time -spent with large data structures, hence it's best to chunk these. - -Basically, unit of work is a large degree of automation in order to -automate the task of persisting a complex object graph into a -relational database with no explicit persistence code, and this -automation has a price. - -ORMs are basically not intended for high-performance bulk inserts - -this is the whole reason SQLAlchemy offers the Core in addition to the -ORM as a first-class component. - -For the use case of fast bulk inserts, the -SQL generation and execution system that the ORM builds on top of -is part of the :doc:`Core `. Using this system directly, we can produce an INSERT that -is competitive with using the raw database API directly. - -Alternatively, the SQLAlchemy ORM offers the :ref:`bulk_operations` -suite of methods, which provide hooks into subsections of the unit of -work process in order to emit Core-level INSERT and UPDATE constructs with -a small degree of ORM-based automation. - -The example below illustrates time-based tests for several different -methods of inserting rows, going from the most automated to the least. -With cPython 2.7, runtimes observed:: - - classics-MacBook-Pro:sqlalchemy classic$ python test.py - SQLAlchemy ORM: Total time for 100000 records 12.0471920967 secs - SQLAlchemy ORM pk given: Total time for 100000 records 7.06283402443 secs - SQLAlchemy ORM bulk_save_objects(): Total time for 100000 records 0.856323003769 secs - SQLAlchemy Core: Total time for 100000 records 0.485800027847 secs - sqlite3: Total time for 100000 records 0.487842082977 sec - -We can reduce the time by a factor of three using recent versions of `Pypy `_:: - - classics-MacBook-Pro:sqlalchemy classic$ /usr/local/src/pypy-2.1-beta2-osx64/bin/pypy test.py - SQLAlchemy ORM: Total time for 100000 records 5.88369488716 secs - SQLAlchemy ORM pk given: Total time for 100000 records 3.52294301987 secs - SQLAlchemy Core: Total time for 100000 records 0.613556146622 secs - sqlite3: Total time for 100000 records 0.442467927933 sec - -Script:: - - import time - import sqlite3 - - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy import Column, Integer, String, create_engine - from sqlalchemy.orm import scoped_session, sessionmaker - - Base = declarative_base() - DBSession = scoped_session(sessionmaker()) - engine = None - - - class Customer(Base): - __tablename__ = "customer" - id = Column(Integer, primary_key=True) - name = Column(String(255)) - - - def init_sqlalchemy(dbname='sqlite:///sqlalchemy.db'): - global engine - engine = create_engine(dbname, echo=False) - DBSession.remove() - DBSession.configure(bind=engine, autoflush=False, expire_on_commit=False) - Base.metadata.drop_all(engine) - Base.metadata.create_all(engine) - - - def test_sqlalchemy_orm(n=100000): - init_sqlalchemy() - t0 = time.time() - for i in xrange(n): - customer = Customer() - customer.name = 'NAME ' + str(i) - DBSession.add(customer) - if i % 1000 == 0: - DBSession.flush() - DBSession.commit() - print( - "SQLAlchemy ORM: Total time for " + str(n) + - " records " + str(time.time() - t0) + " secs") - - - def test_sqlalchemy_orm_pk_given(n=100000): - init_sqlalchemy() - t0 = time.time() - for i in xrange(n): - customer = Customer(id=i+1, name="NAME " + str(i)) - DBSession.add(customer) - if i % 1000 == 0: - DBSession.flush() - DBSession.commit() - print( - "SQLAlchemy ORM pk given: Total time for " + str(n) + - " records " + str(time.time() - t0) + " secs") - - - def test_sqlalchemy_orm_bulk_insert(n=100000): - init_sqlalchemy() - t0 = time.time() - n1 = n - while n1 > 0: - n1 = n1 - 10000 - DBSession.bulk_insert_mappings( - Customer, - [ - dict(name="NAME " + str(i)) - for i in xrange(min(10000, n1)) - ] - ) - DBSession.commit() - print( - "SQLAlchemy ORM bulk_save_objects(): Total time for " + str(n) + - " records " + str(time.time() - t0) + " secs") - - - def test_sqlalchemy_core(n=100000): - init_sqlalchemy() - t0 = time.time() - engine.execute( - Customer.__table__.insert(), - [{"name": 'NAME ' + str(i)} for i in xrange(n)] - ) - print( - "SQLAlchemy Core: Total time for " + str(n) + - " records " + str(time.time() - t0) + " secs") - - - def init_sqlite3(dbname): - conn = sqlite3.connect(dbname) - c = conn.cursor() - c.execute("DROP TABLE IF EXISTS customer") - c.execute( - "CREATE TABLE customer (id INTEGER NOT NULL, " - "name VARCHAR(255), PRIMARY KEY(id))") - conn.commit() - return conn - - - def test_sqlite3(n=100000, dbname='sqlite3.db'): - conn = init_sqlite3(dbname) - c = conn.cursor() - t0 = time.time() - for i in xrange(n): - row = ('NAME ' + str(i),) - c.execute("INSERT INTO customer (name) VALUES (?)", row) - conn.commit() - print( - "sqlite3: Total time for " + str(n) + - " records " + str(time.time() - t0) + " sec") - - if __name__ == '__main__': - test_sqlalchemy_orm(100000) - test_sqlalchemy_orm_pk_given(100000) - test_sqlalchemy_orm_bulk_insert(100000) - test_sqlalchemy_core(100000) - test_sqlite3(100000) - - -Sessions / Queries -=================== - - -"This Session's transaction has been rolled back due to a previous exception during flush." (or similar) ---------------------------------------------------------------------------------------------------------- - -This is an error that occurs when a :meth:`.Session.flush` raises an exception, rolls back -the transaction, but further commands upon the `Session` are called without an -explicit call to :meth:`.Session.rollback` or :meth:`.Session.close`. - -It usually corresponds to an application that catches an exception -upon :meth:`.Session.flush` or :meth:`.Session.commit` and -does not properly handle the exception. For example:: - - from sqlalchemy import create_engine, Column, Integer - from sqlalchemy.orm import sessionmaker - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base(create_engine('sqlite://')) - - class Foo(Base): - __tablename__ = 'foo' - id = Column(Integer, primary_key=True) - - Base.metadata.create_all() - - session = sessionmaker()() - - # constraint violation - session.add_all([Foo(id=1), Foo(id=1)]) - - try: - session.commit() - except: - # ignore error - pass - - # continue using session without rolling back - session.commit() - - -The usage of the :class:`.Session` should fit within a structure similar to this:: - - try: - - session.commit() - except: - session.rollback() - raise - finally: - session.close() # optional, depends on use case - -Many things can cause a failure within the try/except besides flushes. You -should always have some kind of "framing" of your session operations so that -connection and transaction resources have a definitive boundary, otherwise -your application doesn't really have its usage of resources under control. -This is not to say that you need to put try/except blocks all throughout your -application - on the contrary, this would be a terrible idea. You should -architect your application such that there is one (or few) point(s) of -"framing" around session operations. - -For a detailed discussion on how to organize usage of the :class:`.Session`, -please see :ref:`session_faq_whentocreate`. - -But why does flush() insist on issuing a ROLLBACK? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It would be great if :meth:`.Session.flush` could partially complete and then not roll -back, however this is beyond its current capabilities since its internal -bookkeeping would have to be modified such that it can be halted at any time -and be exactly consistent with what's been flushed to the database. While this -is theoretically possible, the usefulness of the enhancement is greatly -decreased by the fact that many database operations require a ROLLBACK in any -case. Postgres in particular has operations which, once failed, the -transaction is not allowed to continue:: - - test=> create table foo(id integer primary key); - NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "foo_pkey" for table "foo" - CREATE TABLE - test=> begin; - BEGIN - test=> insert into foo values(1); - INSERT 0 1 - test=> commit; - COMMIT - test=> begin; - BEGIN - test=> insert into foo values(1); - ERROR: duplicate key value violates unique constraint "foo_pkey" - test=> insert into foo values(2); - ERROR: current transaction is aborted, commands ignored until end of transaction block - -What SQLAlchemy offers that solves both issues is support of SAVEPOINT, via -:meth:`.Session.begin_nested`. Using :meth:`.Session.begin_nested`, you can frame an operation that may -potentially fail within a transaction, and then "roll back" to the point -before its failure while maintaining the enclosing transaction. - -But why isn't the one automatic call to ROLLBACK enough? Why must I ROLLBACK again? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This is again a matter of the :class:`.Session` providing a consistent interface and -refusing to guess about what context its being used. For example, the -:class:`.Session` supports "framing" above within multiple levels. Such as, suppose -you had a decorator ``@with_session()``, which did this:: - - def with_session(fn): - def go(*args, **kw): - session.begin(subtransactions=True) - try: - ret = fn(*args, **kw) - session.commit() - return ret - except: - session.rollback() - raise - return go - -The above decorator begins a transaction if one does not exist already, and -then commits it, if it were the creator. The "subtransactions" flag means that -if :meth:`.Session.begin` were already called by an enclosing function, nothing happens -except a counter is incremented - this counter is decremented when :meth:`.Session.commit` -is called and only when it goes back to zero does the actual COMMIT happen. It -allows this usage pattern:: - - @with_session - def one(): - # do stuff - two() - - - @with_session - def two(): - # etc. - - one() - - two() - -``one()`` can call ``two()``, or ``two()`` can be called by itself, and the -``@with_session`` decorator ensures the appropriate "framing" - the transaction -boundaries stay on the outermost call level. As you can see, if ``two()`` calls -``flush()`` which throws an exception and then issues a ``rollback()``, there will -*always* be a second ``rollback()`` performed by the decorator, and possibly a -third corresponding to two levels of decorator. If the ``flush()`` pushed the -``rollback()`` all the way out to the top of the stack, and then we said that -all remaining ``rollback()`` calls are moot, there is some silent behavior going -on there. A poorly written enclosing method might suppress the exception, and -then call ``commit()`` assuming nothing is wrong, and then you have a silent -failure condition. The main reason people get this error in fact is because -they didn't write clean "framing" code and they would have had other problems -down the road. - -If you think the above use case is a little exotic, the same kind of thing -comes into play if you want to SAVEPOINT- you might call ``begin_nested()`` -several times, and the ``commit()``/``rollback()`` calls each resolve the most -recent ``begin_nested()``. The meaning of ``rollback()`` or ``commit()`` is -dependent upon which enclosing block it is called, and you might have any -sequence of ``rollback()``/``commit()`` in any order, and its the level of nesting -that determines their behavior. - -In both of the above cases, if ``flush()`` broke the nesting of transaction -blocks, the behavior is, depending on scenario, anywhere from "magic" to -silent failure to blatant interruption of code flow. - -``flush()`` makes its own "subtransaction", so that a transaction is started up -regardless of the external transactional state, and when complete it calls -``commit()``, or ``rollback()`` upon failure - but that ``rollback()`` corresponds -to its own subtransaction - it doesn't want to guess how you'd like to handle -the external "framing" of the transaction, which could be nested many levels -with any combination of subtransactions and real SAVEPOINTs. The job of -starting/ending the "frame" is kept consistently with the code external to the -``flush()``, and we made a decision that this was the most consistent approach. - - - -How do I make a Query that always adds a certain filter to every query? ------------------------------------------------------------------------------------------------- - -See the recipe at `PreFilteredQuery `_. - -I've created a mapping against an Outer Join, and while the query returns rows, no objects are returned. Why not? ------------------------------------------------------------------------------------------------------------------- - -Rows returned by an outer join may contain NULL for part of the primary key, -as the primary key is the composite of both tables. The :class:`.Query` object ignores incoming rows -that don't have an acceptable primary key. Based on the setting of the ``allow_partial_pks`` -flag on :func:`.mapper`, a primary key is accepted if the value has at least one non-NULL -value, or alternatively if the value has no NULL values. See ``allow_partial_pks`` -at :func:`.mapper`. - - -I'm using ``joinedload()`` or ``lazy=False`` to create a JOIN/OUTER JOIN and SQLAlchemy is not constructing the correct query when I try to add a WHERE, ORDER BY, LIMIT, etc. (which relies upon the (OUTER) JOIN) ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - -The joins generated by joined eager loading are only used to fully load related -collections, and are designed to have no impact on the primary results of the query. -Since they are anonymously aliased, they cannot be referenced directly. - -For detail on this beahvior, see :doc:`orm/loading`. - -Query has no ``__len__()``, why not? ------------------------------------- - -The Python ``__len__()`` magic method applied to an object allows the ``len()`` -builtin to be used to determine the length of the collection. It's intuitive -that a SQL query object would link ``__len__()`` to the :meth:`.Query.count` -method, which emits a `SELECT COUNT`. The reason this is not possible is -because evaluating the query as a list would incur two SQL calls instead of -one:: - - class Iterates(object): - def __len__(self): - print "LEN!" - return 5 - - def __iter__(self): - print "ITER!" - return iter([1, 2, 3, 4, 5]) - - list(Iterates()) - -output:: - - ITER! - LEN! - -How Do I use Textual SQL with ORM Queries? -------------------------------------------- - -See: - -* :ref:`orm_tutorial_literal_sql` - Ad-hoc textual blocks with :class:`.Query` - -* :ref:`session_sql_expressions` - Using :class:`.Session` with textual SQL directly. - -I'm calling ``Session.delete(myobject)`` and it isn't removed from the parent collection! ------------------------------------------------------------------------------------------- - -See :ref:`session_deleting_from_collections` for a description of this behavior. - -why isn't my ``__init__()`` called when I load objects? -------------------------------------------------------- - -See :ref:`mapping_constructors` for a description of this behavior. - -how do I use ON DELETE CASCADE with SA's ORM? ----------------------------------------------- - -SQLAlchemy will always issue UPDATE or DELETE statements for dependent -rows which are currently loaded in the :class:`.Session`. For rows which -are not loaded, it will by default issue SELECT statements to load -those rows and udpate/delete those as well; in other words it assumes -there is no ON DELETE CASCADE configured. -To configure SQLAlchemy to cooperate with ON DELETE CASCADE, see -:ref:`passive_deletes`. - -I set the "foo_id" attribute on my instance to "7", but the "foo" attribute is still ``None`` - shouldn't it have loaded Foo with id #7? ----------------------------------------------------------------------------------------------------------------------------------------------------- - -The ORM is not constructed in such a way as to support -immediate population of relationships driven from foreign -key attribute changes - instead, it is designed to work the -other way around - foreign key attributes are handled by the -ORM behind the scenes, the end user sets up object -relationships naturally. Therefore, the recommended way to -set ``o.foo`` is to do just that - set it!:: - - foo = Session.query(Foo).get(7) - o.foo = foo - Session.commit() - -Manipulation of foreign key attributes is of course entirely legal. However, -setting a foreign-key attribute to a new value currently does not trigger -an "expire" event of the :func:`.relationship` in which it's involved. This means -that for the following sequence:: - - o = Session.query(SomeClass).first() - assert o.foo is None # accessing an un-set attribute sets it to None - o.foo_id = 7 - -``o.foo`` is initialized to ``None`` when we first accessed it. Setting -``o.foo_id = 7`` will have the value of "7" as pending, but no flush -has occurred - so ``o.foo`` is still ``None``:: - - # attribute is already set to None, has not been - # reconciled with o.foo_id = 7 yet - assert o.foo is None - -For ``o.foo`` to load based on the foreign key mutation is usually achieved -naturally after the commit, which both flushes the new foreign key value -and expires all state:: - - Session.commit() # expires all attributes - - foo_7 = Session.query(Foo).get(7) - - assert o.foo is foo_7 # o.foo lazyloads on access - -A more minimal operation is to expire the attribute individually - this can -be performed for any :term:`persistent` object using :meth:`.Session.expire`:: - - o = Session.query(SomeClass).first() - o.foo_id = 7 - Session.expire(o, ['foo']) # object must be persistent for this - - foo_7 = Session.query(Foo).get(7) - - assert o.foo is foo_7 # o.foo lazyloads on access - -Note that if the object is not persistent but present in the :class:`.Session`, -it's known as :term:`pending`. This means the row for the object has not been -INSERTed into the database yet. For such an object, setting ``foo_id`` does not -have meaning until the row is inserted; otherwise there is no row yet:: - - new_obj = SomeClass() - new_obj.foo_id = 7 - - Session.add(new_obj) - - # accessing an un-set attribute sets it to None - assert new_obj.foo is None - - Session.flush() # emits INSERT - - # expire this because we already set .foo to None - Session.expire(o, ['foo']) - - assert new_obj.foo is foo_7 # now it loads - - -.. topic:: Attribute loading for non-persistent objects - - One variant on the "pending" behavior above is if we use the flag - ``load_on_pending`` on :func:`.relationship`. When this flag is set, the - lazy loader will emit for ``new_obj.foo`` before the INSERT proceeds; another - variant of this is to use the :meth:`.Session.enable_relationship_loading` - method, which can "attach" an object to a :class:`.Session` in such a way that - many-to-one relationships load as according to foreign key attributes - regardless of the object being in any particular state. - Both techniques are **not recommended for general use**; they were added to suit - specific programming scenarios encountered by users which involve the repurposing - of the ORM's usual object states. - -The recipe `ExpireRelationshipOnFKChange `_ features an example using SQLAlchemy events -in order to coordinate the setting of foreign key attributes with many-to-one -relationships. - -Is there a way to automagically have only unique keywords (or other kinds of objects) without doing a query for the keyword and getting a reference to the row containing that keyword? ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - -When people read the many-to-many example in the docs, they get hit with the -fact that if you create the same ``Keyword`` twice, it gets put in the DB twice. -Which is somewhat inconvenient. - -This `UniqueObject `_ recipe was created to address this issue. - - diff --git a/doc/build/faq/connections.rst b/doc/build/faq/connections.rst new file mode 100644 index 000000000..81a8678b4 --- /dev/null +++ b/doc/build/faq/connections.rst @@ -0,0 +1,138 @@ +Connections / Engines +===================== + +.. contents:: + :local: + :class: faq + :backlinks: none + + +How do I configure logging? +--------------------------- + +See :ref:`dbengine_logging`. + +How do I pool database connections? Are my connections pooled? +---------------------------------------------------------------- + +SQLAlchemy performs application-level connection pooling automatically +in most cases. With the exception of SQLite, a :class:`.Engine` object +refers to a :class:`.QueuePool` as a source of connectivity. + +For more detail, see :ref:`engines_toplevel` and :ref:`pooling_toplevel`. + +How do I pass custom connect arguments to my database API? +----------------------------------------------------------- + +The :func:`.create_engine` call accepts additional arguments either +directly via the ``connect_args`` keyword argument:: + + e = create_engine("mysql://scott:tiger@localhost/test", + connect_args={"encoding": "utf8"}) + +Or for basic string and integer arguments, they can usually be specified +in the query string of the URL:: + + e = create_engine("mysql://scott:tiger@localhost/test?encoding=utf8") + +.. seealso:: + + :ref:`custom_dbapi_args` + +"MySQL Server has gone away" +---------------------------- + +There are two major causes for this error: + +1. The MySQL client closes connections which have been idle for a set period +of time, defaulting to eight hours. This can be avoided by using the ``pool_recycle`` +setting with :func:`.create_engine`, described at :ref:`mysql_connection_timeouts`. + +2. Usage of the MySQLdb :term:`DBAPI`, or a similar DBAPI, in a non-threadsafe manner, or in an otherwise +inappropriate way. The MySQLdb connection object is not threadsafe - this expands +out to any SQLAlchemy system that links to a single connection, which includes the ORM +:class:`.Session`. For background +on how :class:`.Session` should be used in a multithreaded environment, +see :ref:`session_faq_threadsafe`. + +Why does SQLAlchemy issue so many ROLLBACKs? +--------------------------------------------- + +SQLAlchemy currently assumes DBAPI connections are in "non-autocommit" mode - +this is the default behavior of the Python database API, meaning it +must be assumed that a transaction is always in progress. The +connection pool issues ``connection.rollback()`` when a connection is returned. +This is so that any transactional resources remaining on the connection are +released. On a database like Postgresql or MSSQL where table resources are +aggressively locked, this is critical so that rows and tables don't remain +locked within connections that are no longer in use. An application can +otherwise hang. It's not just for locks, however, and is equally critical on +any database that has any kind of transaction isolation, including MySQL with +InnoDB. Any connection that is still inside an old transaction will return +stale data, if that data was already queried on that connection within +isolation. For background on why you might see stale data even on MySQL, see +http://dev.mysql.com/doc/refman/5.1/en/innodb-transaction-model.html + +I'm on MyISAM - how do I turn it off? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The behavior of the connection pool's connection return behavior can be +configured using ``reset_on_return``:: + + from sqlalchemy import create_engine + from sqlalchemy.pool import QueuePool + + engine = create_engine('mysql://scott:tiger@localhost/myisam_database', pool=QueuePool(reset_on_return=False)) + +I'm on SQL Server - how do I turn those ROLLBACKs into COMMITs? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``reset_on_return`` accepts the values ``commit``, ``rollback`` in addition +to ``True``, ``False``, and ``None``. Setting to ``commit`` will cause +a COMMIT as any connection is returned to the pool:: + + engine = create_engine('mssql://scott:tiger@mydsn', pool=QueuePool(reset_on_return='commit')) + + +I am using multiple connections with a SQLite database (typically to test transaction operation), and my test program is not working! +---------------------------------------------------------------------------------------------------------------------------------------------------------- + +If using a SQLite ``:memory:`` database, or a version of SQLAlchemy prior +to version 0.7, the default connection pool is the :class:`.SingletonThreadPool`, +which maintains exactly one SQLite connection per thread. So two +connections in use in the same thread will actually be the same SQLite +connection. Make sure you're not using a :memory: database and +use :class:`.NullPool`, which is the default for non-memory databases in +current SQLAlchemy versions. + +.. seealso:: + + :ref:`pysqlite_threading_pooling` - info on PySQLite's behavior. + +How do I get at the raw DBAPI connection when using an Engine? +-------------------------------------------------------------- + +With a regular SA engine-level Connection, you can get at a pool-proxied +version of the DBAPI connection via the :attr:`.Connection.connection` attribute on +:class:`.Connection`, and for the really-real DBAPI connection you can call the +:attr:`.ConnectionFairy.connection` attribute on that - but there should never be any need to access +the non-pool-proxied DBAPI connection, as all methods are proxied through:: + + engine = create_engine(...) + conn = engine.connect() + conn.connection. + cursor = conn.connection.cursor() + +You must ensure that you revert any isolation level settings or other +operation-specific settings on the connection back to normal before returning +it to the pool. + +As an alternative to reverting settings, you can call the :meth:`.Connection.detach` method on +either :class:`.Connection` or the proxied connection, which will de-associate +the connection from the pool such that it will be closed and discarded +when :meth:`.Connection.close` is called:: + + conn = engine.connect() + conn.detach() # detaches the DBAPI connection from the connection pool + conn.connection. + conn.close() # connection is closed for real, the pool replaces it with a new connection diff --git a/doc/build/faq/index.rst b/doc/build/faq/index.rst new file mode 100644 index 000000000..120e0ba3a --- /dev/null +++ b/doc/build/faq/index.rst @@ -0,0 +1,19 @@ +.. _faq_toplevel: + +============================ +Frequently Asked Questions +============================ + +The Frequently Asked Questions section is a growing collection of commonly +observed questions to well-known issues. + +.. toctree:: + :maxdepth: 1 + + connections + metadata_schema + sqlexpressions + ormconfiguration + performance + sessions + diff --git a/doc/build/faq/metadata_schema.rst b/doc/build/faq/metadata_schema.rst new file mode 100644 index 000000000..9697399dc --- /dev/null +++ b/doc/build/faq/metadata_schema.rst @@ -0,0 +1,102 @@ +================== +MetaData / Schema +================== + +.. contents:: + :local: + :class: faq + :backlinks: none + + + +My program is hanging when I say ``table.drop()`` / ``metadata.drop_all()`` +=========================================================================== + +This usually corresponds to two conditions: 1. using PostgreSQL, which is really +strict about table locks, and 2. you have a connection still open which +contains locks on the table and is distinct from the connection being used for +the DROP statement. Heres the most minimal version of the pattern:: + + connection = engine.connect() + result = connection.execute(mytable.select()) + + mytable.drop(engine) + +Above, a connection pool connection is still checked out; furthermore, the +result object above also maintains a link to this connection. If +"implicit execution" is used, the result will hold this connection opened until +the result object is closed or all rows are exhausted. + +The call to ``mytable.drop(engine)`` attempts to emit DROP TABLE on a second +connection procured from the :class:`.Engine` which will lock. + +The solution is to close out all connections before emitting DROP TABLE:: + + connection = engine.connect() + result = connection.execute(mytable.select()) + + # fully read result sets + result.fetchall() + + # close connections + connection.close() + + # now locks are removed + mytable.drop(engine) + +Does SQLAlchemy support ALTER TABLE, CREATE VIEW, CREATE TRIGGER, Schema Upgrade Functionality? +=============================================================================================== + + +General ALTER support isn't present in SQLAlchemy directly. For special DDL +on an ad-hoc basis, the :class:`.DDL` and related constructs can be used. +See :doc:`core/ddl` for a discussion on this subject. + +A more comprehensive option is to use schema migration tools, such as Alembic +or SQLAlchemy-Migrate; see :ref:`schema_migrations` for discussion on this. + +How can I sort Table objects in order of their dependency? +=========================================================================== + +This is available via the :attr:`.MetaData.sorted_tables` function:: + + metadata = MetaData() + # ... add Table objects to metadata + ti = metadata.sorted_tables: + for t in ti: + print t + +How can I get the CREATE TABLE/ DROP TABLE output as a string? +=========================================================================== + +Modern SQLAlchemy has clause constructs which represent DDL operations. These +can be rendered to strings like any other SQL expression:: + + from sqlalchemy.schema import CreateTable + + print CreateTable(mytable) + +To get the string specific to a certain engine:: + + print CreateTable(mytable).compile(engine) + +There's also a special form of :class:`.Engine` that can let you dump an entire +metadata creation sequence, using this recipe:: + + def dump(sql, *multiparams, **params): + print sql.compile(dialect=engine.dialect) + engine = create_engine('postgresql://', strategy='mock', executor=dump) + metadata.create_all(engine, checkfirst=False) + +The `Alembic `_ tool also supports +an "offline" SQL generation mode that renders database migrations as SQL scripts. + +How can I subclass Table/Column to provide certain behaviors/configurations? +============================================================================= + +:class:`.Table` and :class:`.Column` are not good targets for direct subclassing. +However, there are simple ways to get on-construction behaviors using creation +functions, and behaviors related to the linkages between schema objects such as +constraint conventions or naming conventions using attachment events. +An example of many of these +techniques can be seen at `Naming Conventions `_. diff --git a/doc/build/faq/ormconfiguration.rst b/doc/build/faq/ormconfiguration.rst new file mode 100644 index 000000000..3a2ea29a6 --- /dev/null +++ b/doc/build/faq/ormconfiguration.rst @@ -0,0 +1,334 @@ +ORM Configuration +================== + +.. contents:: + :local: + :class: faq + :backlinks: none + +.. _faq_mapper_primary_key: + +How do I map a table that has no primary key? +--------------------------------------------- + +The SQLAlchemy ORM, in order to map to a particular table, needs there to be +at least one column denoted as a primary key column; multiple-column, +i.e. composite, primary keys are of course entirely feasible as well. These +columns do **not** need to be actually known to the database as primary key +columns, though it's a good idea that they are. It's only necessary that the columns +*behave* as a primary key does, e.g. as a unique and not nullable identifier +for a row. + +Most ORMs require that objects have some kind of primary key defined +because the object in memory must correspond to a uniquely identifiable +row in the database table; at the very least, this allows the +object can be targeted for UPDATE and DELETE statements which will affect only +that object's row and no other. However, the importance of the primary key +goes far beyond that. In SQLAlchemy, all ORM-mapped objects are at all times +linked uniquely within a :class:`.Session` +to their specific database row using a pattern called the :term:`identity map`, +a pattern that's central to the unit of work system employed by SQLAlchemy, +and is also key to the most common (and not-so-common) patterns of ORM usage. + + +.. note:: + + It's important to note that we're only talking about the SQLAlchemy ORM; an + application which builds on Core and deals only with :class:`.Table` objects, + :func:`.select` constructs and the like, **does not** need any primary key + to be present on or associated with a table in any way (though again, in SQL, all tables + should really have some kind of primary key, lest you need to actually + update or delete specific rows). + +In almost all cases, a table does have a so-called :term:`candidate key`, which is a column or series +of columns that uniquely identify a row. If a table truly doesn't have this, and has actual +fully duplicate rows, the table is not corresponding to `first normal form `_ and cannot be mapped. Otherwise, whatever columns comprise the best candidate key can be +applied directly to the mapper:: + + class SomeClass(Base): + __table__ = some_table_with_no_pk + __mapper_args__ = { + 'primary_key':[some_table_with_no_pk.c.uid, some_table_with_no_pk.c.bar] + } + +Better yet is when using fully declared table metadata, use the ``primary_key=True`` +flag on those columns:: + + class SomeClass(Base): + __tablename__ = "some_table_with_no_pk" + + uid = Column(Integer, primary_key=True) + bar = Column(String, primary_key=True) + +All tables in a relational database should have primary keys. Even a many-to-many +association table - the primary key would be the composite of the two association +columns:: + + CREATE TABLE my_association ( + user_id INTEGER REFERENCES user(id), + account_id INTEGER REFERENCES account(id), + PRIMARY KEY (user_id, account_id) + ) + + +How do I configure a Column that is a Python reserved word or similar? +---------------------------------------------------------------------------- + +Column-based attributes can be given any name desired in the mapping. See +:ref:`mapper_column_distinct_names`. + +How do I get a list of all columns, relationships, mapped attributes, etc. given a mapped class? +------------------------------------------------------------------------------------------------- + +This information is all available from the :class:`.Mapper` object. + +To get at the :class:`.Mapper` for a particular mapped class, call the +:func:`.inspect` function on it:: + + from sqlalchemy import inspect + + mapper = inspect(MyClass) + +From there, all information about the class can be acquired using such methods as: + +* :attr:`.Mapper.attrs` - a namespace of all mapped attributes. The attributes + themselves are instances of :class:`.MapperProperty`, which contain additional + attributes that can lead to the mapped SQL expression or column, if applicable. + +* :attr:`.Mapper.column_attrs` - the mapped attribute namespace + limited to column and SQL expression attributes. You might want to use + :attr:`.Mapper.columns` to get at the :class:`.Column` objects directly. + +* :attr:`.Mapper.relationships` - namespace of all :class:`.RelationshipProperty` attributes. + +* :attr:`.Mapper.all_orm_descriptors` - namespace of all mapped attributes, plus user-defined + attributes defined using systems such as :class:`.hybrid_property`, :class:`.AssociationProxy` and others. + +* :attr:`.Mapper.columns` - A namespace of :class:`.Column` objects and other named + SQL expressions associated with the mapping. + +* :attr:`.Mapper.mapped_table` - The :class:`.Table` or other selectable to which + this mapper is mapped. + +* :attr:`.Mapper.local_table` - The :class:`.Table` that is "local" to this mapper; + this differs from :attr:`.Mapper.mapped_table` in the case of a mapper mapped + using inheritance to a composed selectable. + +.. _faq_combining_columns: + +I'm getting a warning or error about "Implicitly combining column X under attribute Y" +-------------------------------------------------------------------------------------- + +This condition refers to when a mapping contains two columns that are being +mapped under the same attribute name due to their name, but there's no indication +that this is intentional. A mapped class needs to have explicit names for +every attribute that is to store an independent value; when two columns have the +same name and aren't disambiguated, they fall under the same attribute and +the effect is that the value from one column is **copied** into the other, based +on which column was assigned to the attribute first. + +This behavior is often desirable and is allowed without warning in the case +where the two columns are linked together via a foreign key relationship +within an inheritance mapping. When the warning or exception occurs, the +issue can be resolved by either assigning the columns to differently-named +attributes, or if combining them together is desired, by using +:func:`.column_property` to make this explicit. + +Given the example as follows:: + + from sqlalchemy import Integer, Column, ForeignKey + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + class A(Base): + __tablename__ = 'a' + + id = Column(Integer, primary_key=True) + + class B(A): + __tablename__ = 'b' + + id = Column(Integer, primary_key=True) + a_id = Column(Integer, ForeignKey('a.id')) + +As of SQLAlchemy version 0.9.5, the above condition is detected, and will +warn that the ``id`` column of ``A`` and ``B`` is being combined under +the same-named attribute ``id``, which above is a serious issue since it means +that a ``B`` object's primary key will always mirror that of its ``A``. + +A mapping which resolves this is as follows:: + + class A(Base): + __tablename__ = 'a' + + id = Column(Integer, primary_key=True) + + class B(A): + __tablename__ = 'b' + + b_id = Column('id', Integer, primary_key=True) + a_id = Column(Integer, ForeignKey('a.id')) + +Suppose we did want ``A.id`` and ``B.id`` to be mirrors of each other, despite +the fact that ``B.a_id`` is where ``A.id`` is related. We could combine +them together using :func:`.column_property`:: + + class A(Base): + __tablename__ = 'a' + + id = Column(Integer, primary_key=True) + + class B(A): + __tablename__ = 'b' + + # probably not what you want, but this is a demonstration + id = column_property(Column(Integer, primary_key=True), A.id) + a_id = Column(Integer, ForeignKey('a.id')) + + + +I'm using Declarative and setting primaryjoin/secondaryjoin using an ``and_()`` or ``or_()``, and I am getting an error message about foreign keys. +------------------------------------------------------------------------------------------------------------------------------------------------------------------ + +Are you doing this?:: + + class MyClass(Base): + # .... + + foo = relationship("Dest", primaryjoin=and_("MyClass.id==Dest.foo_id", "MyClass.foo==Dest.bar")) + +That's an ``and_()`` of two string expressions, which SQLAlchemy cannot apply any mapping towards. Declarative allows :func:`.relationship` arguments to be specified as strings, which are converted into expression objects using ``eval()``. But this doesn't occur inside of an ``and_()`` expression - it's a special operation declarative applies only to the *entirety* of what's passed to primaryjoin or other arguments as a string:: + + class MyClass(Base): + # .... + + foo = relationship("Dest", primaryjoin="and_(MyClass.id==Dest.foo_id, MyClass.foo==Dest.bar)") + +Or if the objects you need are already available, skip the strings:: + + class MyClass(Base): + # .... + + foo = relationship(Dest, primaryjoin=and_(MyClass.id==Dest.foo_id, MyClass.foo==Dest.bar)) + +The same idea applies to all the other arguments, such as ``foreign_keys``:: + + # wrong ! + foo = relationship(Dest, foreign_keys=["Dest.foo_id", "Dest.bar_id"]) + + # correct ! + foo = relationship(Dest, foreign_keys="[Dest.foo_id, Dest.bar_id]") + + # also correct ! + foo = relationship(Dest, foreign_keys=[Dest.foo_id, Dest.bar_id]) + + # if you're using columns from the class that you're inside of, just use the column objects ! + class MyClass(Base): + foo_id = Column(...) + bar_id = Column(...) + # ... + + foo = relationship(Dest, foreign_keys=[foo_id, bar_id]) + +.. _faq_subqueryload_limit_sort: + +Why is ``ORDER BY`` required with ``LIMIT`` (especially with ``subqueryload()``)? +--------------------------------------------------------------------------------- + +A relational database can return rows in any +arbitrary order, when an explicit ordering is not set. +While this ordering very often corresponds to the natural +order of rows within a table, this is not the case for all databases and +all queries. The consequence of this is that any query that limits rows +using ``LIMIT`` or ``OFFSET`` should **always** specify an ``ORDER BY``. +Otherwise, it is not deterministic which rows will actually be returned. + +When we use a SQLAlchemy method like :meth:`.Query.first`, we are in fact +applying a ``LIMIT`` of one to the query, so without an explicit ordering +it is not deterministic what row we actually get back. +While we may not notice this for simple queries on databases that usually +returns rows in their natural +order, it becomes much more of an issue if we also use :func:`.orm.subqueryload` +to load related collections, and we may not be loading the collections +as intended. + +SQLAlchemy implements :func:`.orm.subqueryload` by issuing a separate query, +the results of which are matched up to the results from the first query. +We see two queries emitted like this: + +.. sourcecode:: python+sql + + >>> session.query(User).options(subqueryload(User.addresses)).all() + {opensql}-- the "main" query + SELECT users.id AS users_id + FROM users + {stop} + {opensql}-- the "load" query issued by subqueryload + SELECT addresses.id AS addresses_id, + addresses.user_id AS addresses_user_id, + anon_1.users_id AS anon_1_users_id + FROM (SELECT users.id AS users_id FROM users) AS anon_1 + JOIN addresses ON anon_1.users_id = addresses.user_id + ORDER BY anon_1.users_id + +The second query embeds the first query as a source of rows. +When the inner query uses ``OFFSET`` and/or ``LIMIT`` without ordering, +the two queries may not see the same results: + +.. sourcecode:: python+sql + + >>> user = session.query(User).options(subqueryload(User.addresses)).first() + {opensql}-- the "main" query + SELECT users.id AS users_id + FROM users + LIMIT 1 + {stop} + {opensql}-- the "load" query issued by subqueryload + SELECT addresses.id AS addresses_id, + addresses.user_id AS addresses_user_id, + anon_1.users_id AS anon_1_users_id + FROM (SELECT users.id AS users_id FROM users LIMIT 1) AS anon_1 + JOIN addresses ON anon_1.users_id = addresses.user_id + ORDER BY anon_1.users_id + +Depending on database specifics, there is +a chance we may get the a result like the following for the two queries:: + + -- query #1 + +--------+ + |users_id| + +--------+ + | 1| + +--------+ + + -- query #2 + +------------+-----------------+---------------+ + |addresses_id|addresses_user_id|anon_1_users_id| + +------------+-----------------+---------------+ + | 3| 2| 2| + +------------+-----------------+---------------+ + | 4| 2| 2| + +------------+-----------------+---------------+ + +Above, we receive two ``addresses`` rows for ``user.id`` of 2, and none for +1. We've wasted two rows and failed to actually load the collection. This +is an insidious error because without looking at the SQL and the results, the +ORM will not show that there's any issue; if we access the ``addresses`` +for the ``User`` we have, it will emit a lazy load for the collection and we +won't see that anything actually went wrong. + +The solution to this problem is to always specify a deterministic sort order, +so that the main query always returns the same set of rows. This generally +means that you should :meth:`.Query.order_by` on a unique column on the table. +The primary key is a good choice for this:: + + session.query(User).options(subqueryload(User.addresses)).order_by(User.id).first() + +Note that :func:`.joinedload` does not suffer from the same problem because +only one query is ever issued, so the load query cannot be different from the +main query. + +.. seealso:: + + :ref:`subqueryload_ordering` diff --git a/doc/build/faq/performance.rst b/doc/build/faq/performance.rst new file mode 100644 index 000000000..8413cb5a2 --- /dev/null +++ b/doc/build/faq/performance.rst @@ -0,0 +1,443 @@ +.. _faq_performance: + +Performance +=========== + +.. contents:: + :local: + :class: faq + :backlinks: none + +.. _faq_how_to_profile: + +How can I profile a SQLAlchemy powered application? +--------------------------------------------------- + +Looking for performance issues typically involves two stratgies. One +is query profiling, and the other is code profiling. + +Query Profiling +^^^^^^^^^^^^^^^^ + +Sometimes just plain SQL logging (enabled via python's logging module +or via the ``echo=True`` argument on :func:`.create_engine`) can give an +idea how long things are taking. For example, if you log something +right after a SQL operation, you'd see something like this in your +log:: + + 17:37:48,325 INFO [sqlalchemy.engine.base.Engine.0x...048c] SELECT ... + 17:37:48,326 INFO [sqlalchemy.engine.base.Engine.0x...048c] {} + 17:37:48,660 DEBUG [myapp.somemessage] + +if you logged ``myapp.somemessage`` right after the operation, you know +it took 334ms to complete the SQL part of things. + +Logging SQL will also illustrate if dozens/hundreds of queries are +being issued which could be better organized into much fewer queries. +When using the SQLAlchemy ORM, the "eager loading" +feature is provided to partially (:func:`.contains_eager()`) or fully +(:func:`.joinedload()`, :func:`.subqueryload()`) +automate this activity, but without +the ORM "eager loading" typically means to use joins so that results across multiple +tables can be loaded in one result set instead of multiplying numbers +of queries as more depth is added (i.e. ``r + r*r2 + r*r2*r3`` ...) + +For more long-term profiling of queries, or to implement an application-side +"slow query" monitor, events can be used to intercept cursor executions, +using a recipe like the following:: + + from sqlalchemy import event + from sqlalchemy.engine import Engine + import time + import logging + + logging.basicConfig() + logger = logging.getLogger("myapp.sqltime") + logger.setLevel(logging.DEBUG) + + @event.listens_for(Engine, "before_cursor_execute") + def before_cursor_execute(conn, cursor, statement, + parameters, context, executemany): + conn.info.setdefault('query_start_time', []).append(time.time()) + logger.debug("Start Query: %s", statement) + + @event.listens_for(Engine, "after_cursor_execute") + def after_cursor_execute(conn, cursor, statement, + parameters, context, executemany): + total = time.time() - conn.info['query_start_time'].pop(-1) + logger.debug("Query Complete!") + logger.debug("Total Time: %f", total) + +Above, we use the :meth:`.ConnectionEvents.before_cursor_execute` and +:meth:`.ConnectionEvents.after_cursor_execute` events to establish an interception +point around when a statement is executed. We attach a timer onto the +connection using the :class:`._ConnectionRecord.info` dictionary; we use a +stack here for the occasional case where the cursor execute events may be nested. + +Code Profiling +^^^^^^^^^^^^^^ + +If logging reveals that individual queries are taking too long, you'd +need a breakdown of how much time was spent within the database +processing the query, sending results over the network, being handled +by the :term:`DBAPI`, and finally being received by SQLAlchemy's result set +and/or ORM layer. Each of these stages can present their own +individual bottlenecks, depending on specifics. + +For that you need to use the +`Python Profiling Module `_. +Below is a simple recipe which works profiling into a context manager:: + + import cProfile + import StringIO + import pstats + import contextlib + + @contextlib.contextmanager + def profiled(): + pr = cProfile.Profile() + pr.enable() + yield + pr.disable() + s = StringIO.StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps.print_stats() + # uncomment this to see who's calling what + # ps.print_callers() + print s.getvalue() + +To profile a section of code:: + + with profiled(): + Session.query(FooClass).filter(FooClass.somevalue==8).all() + +The output of profiling can be used to give an idea where time is +being spent. A section of profiling output looks like this:: + + 13726 function calls (13042 primitive calls) in 0.014 seconds + + Ordered by: cumulative time + + ncalls tottime percall cumtime percall filename:lineno(function) + 222/21 0.001 0.000 0.011 0.001 lib/sqlalchemy/orm/loading.py:26(instances) + 220/20 0.002 0.000 0.010 0.001 lib/sqlalchemy/orm/loading.py:327(_instance) + 220/20 0.000 0.000 0.010 0.000 lib/sqlalchemy/orm/loading.py:284(populate_state) + 20 0.000 0.000 0.010 0.000 lib/sqlalchemy/orm/strategies.py:987(load_collection_from_subq) + 20 0.000 0.000 0.009 0.000 lib/sqlalchemy/orm/strategies.py:935(get) + 1 0.000 0.000 0.009 0.009 lib/sqlalchemy/orm/strategies.py:940(_load) + 21 0.000 0.000 0.008 0.000 lib/sqlalchemy/orm/strategies.py:942() + 2 0.000 0.000 0.004 0.002 lib/sqlalchemy/orm/query.py:2400(__iter__) + 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/orm/query.py:2414(_execute_and_instances) + 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/engine/base.py:659(execute) + 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/sql/elements.py:321(_execute_on_connection) + 2 0.000 0.000 0.002 0.001 lib/sqlalchemy/engine/base.py:788(_execute_clauseelement) + + ... + +Above, we can see that the ``instances()`` SQLAlchemy function was called 222 +times (recursively, and 21 times from the outside), taking a total of .011 +seconds for all calls combined. + +Execution Slowness +^^^^^^^^^^^^^^^^^^ + +The specifics of these calls can tell us where the time is being spent. +If for example, you see time being spent within ``cursor.execute()``, +e.g. against the DBAPI:: + + 2 0.102 0.102 0.204 0.102 {method 'execute' of 'sqlite3.Cursor' objects} + +this would indicate that the database is taking a long time to start returning +results, and it means your query should be optimized, either by adding indexes +or restructuring the query and/or underlying schema. For that task, +analysis of the query plan is warranted, using a system such as EXPLAIN, +SHOW PLAN, etc. as is provided by the database backend. + +Result Fetching Slowness - Core +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If on the other hand you see many thousands of calls related to fetching rows, +or very long calls to ``fetchall()``, it may +mean your query is returning more rows than expected, or that the fetching +of rows itself is slow. The ORM itself typically uses ``fetchall()`` to fetch +rows (or ``fetchmany()`` if the :meth:`.Query.yield_per` option is used). + +An inordinately large number of rows would be indicated +by a very slow call to ``fetchall()`` at the DBAPI level:: + + 2 0.300 0.600 0.300 0.600 {method 'fetchall' of 'sqlite3.Cursor' objects} + +An unexpectedly large number of rows, even if the ultimate result doesn't seem +to have many rows, can be the result of a cartesian product - when multiple +sets of rows are combined together without appropriately joining the tables +together. It's often easy to produce this behavior with SQLAlchemy Core or +ORM query if the wrong :class:`.Column` objects are used in a complex query, +pulling in additional FROM clauses that are unexpected. + +On the other hand, a fast call to ``fetchall()`` at the DBAPI level, but then +slowness when SQLAlchemy's :class:`.ResultProxy` is asked to do a ``fetchall()``, +may indicate slowness in processing of datatypes, such as unicode conversions +and similar:: + + # the DBAPI cursor is fast... + 2 0.020 0.040 0.020 0.040 {method 'fetchall' of 'sqlite3.Cursor' objects} + + ... + + # but SQLAlchemy's result proxy is slow, this is type-level processing + 2 0.100 0.200 0.100 0.200 lib/sqlalchemy/engine/result.py:778(fetchall) + +In some cases, a backend might be doing type-level processing that isn't +needed. More specifically, seeing calls within the type API that are slow +are better indicators - below is what it looks like when we use a type like +this:: + + from sqlalchemy import TypeDecorator + import time + + class Foo(TypeDecorator): + impl = String + + def process_result_value(self, value, thing): + # intentionally add slowness for illustration purposes + time.sleep(.001) + return value + +the profiling output of this intentionally slow operation can be seen like this:: + + 200 0.001 0.000 0.237 0.001 lib/sqlalchemy/sql/type_api.py:911(process) + 200 0.001 0.000 0.236 0.001 test.py:28(process_result_value) + 200 0.235 0.001 0.235 0.001 {time.sleep} + +that is, we see many expensive calls within the ``type_api`` system, and the actual +time consuming thing is the ``time.sleep()`` call. + +Make sure to check the :doc:`Dialect documentation ` +for notes on known performance tuning suggestions at this level, especially for +databases like Oracle. There may be systems related to ensuring numeric accuracy +or string processing that may not be needed in all cases. + +There also may be even more low-level points at which row-fetching performance is suffering; +for example, if time spent seems to focus on a call like ``socket.receive()``, +that could indicate that everything is fast except for the actual network connection, +and too much time is spent with data moving over the network. + +Result Fetching Slowness - ORM +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To detect slowness in ORM fetching of rows (which is the most common area +of performance concern), calls like ``populate_state()`` and ``_instance()`` will +illustrate individual ORM object populations:: + + # the ORM calls _instance for each ORM-loaded row it sees, and + # populate_state for each ORM-loaded row that results in the population + # of an object's attributes + 220/20 0.001 0.000 0.010 0.000 lib/sqlalchemy/orm/loading.py:327(_instance) + 220/20 0.000 0.000 0.009 0.000 lib/sqlalchemy/orm/loading.py:284(populate_state) + +The ORM's slowness in turning rows into ORM-mapped objects is a product +of the complexity of this operation combined with the overhead of cPython. +Common strategies to mitigate this include: + +* fetch individual columns instead of full entities, that is:: + + session.query(User.id, User.name) + + instead of:: + + session.query(User) + +* Use :class:`.Bundle` objects to organize column-based results:: + + u_b = Bundle('user', User.id, User.name) + a_b = Bundle('address', Address.id, Address.email) + + for user, address in session.query(u_b, a_b).join(User.addresses): + # ... + +* Use result caching - see :ref:`examples_caching` for an in-depth example + of this. + +* Consider a faster interpreter like that of Pypy. + +The output of a profile can be a little daunting but after some +practice they are very easy to read. + +.. seealso:: + + :ref:`examples_performance` - a suite of performance demonstrations + with bundled profiling capabilities. + +I'm inserting 400,000 rows with the ORM and it's really slow! +-------------------------------------------------------------- + +The SQLAlchemy ORM uses the :term:`unit of work` pattern when synchronizing +changes to the database. This pattern goes far beyond simple "inserts" +of data. It includes that attributes which are assigned on objects are +received using an attribute instrumentation system which tracks +changes on objects as they are made, includes that all rows inserted +are tracked in an identity map which has the effect that for each row +SQLAlchemy must retrieve its "last inserted id" if not already given, +and also involves that rows to be inserted are scanned and sorted for +dependencies as needed. Objects are also subject to a fair degree of +bookkeeping in order to keep all of this running, which for a very +large number of rows at once can create an inordinate amount of time +spent with large data structures, hence it's best to chunk these. + +Basically, unit of work is a large degree of automation in order to +automate the task of persisting a complex object graph into a +relational database with no explicit persistence code, and this +automation has a price. + +ORMs are basically not intended for high-performance bulk inserts - +this is the whole reason SQLAlchemy offers the Core in addition to the +ORM as a first-class component. + +For the use case of fast bulk inserts, the +SQL generation and execution system that the ORM builds on top of +is part of the :doc:`Core `. Using this system directly, we can produce an INSERT that +is competitive with using the raw database API directly. + +Alternatively, the SQLAlchemy ORM offers the :ref:`bulk_operations` +suite of methods, which provide hooks into subsections of the unit of +work process in order to emit Core-level INSERT and UPDATE constructs with +a small degree of ORM-based automation. + +The example below illustrates time-based tests for several different +methods of inserting rows, going from the most automated to the least. +With cPython 2.7, runtimes observed:: + + classics-MacBook-Pro:sqlalchemy classic$ python test.py + SQLAlchemy ORM: Total time for 100000 records 12.0471920967 secs + SQLAlchemy ORM pk given: Total time for 100000 records 7.06283402443 secs + SQLAlchemy ORM bulk_save_objects(): Total time for 100000 records 0.856323003769 secs + SQLAlchemy Core: Total time for 100000 records 0.485800027847 secs + sqlite3: Total time for 100000 records 0.487842082977 sec + +We can reduce the time by a factor of three using recent versions of `Pypy `_:: + + classics-MacBook-Pro:sqlalchemy classic$ /usr/local/src/pypy-2.1-beta2-osx64/bin/pypy test.py + SQLAlchemy ORM: Total time for 100000 records 5.88369488716 secs + SQLAlchemy ORM pk given: Total time for 100000 records 3.52294301987 secs + SQLAlchemy Core: Total time for 100000 records 0.613556146622 secs + sqlite3: Total time for 100000 records 0.442467927933 sec + +Script:: + + import time + import sqlite3 + + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy import Column, Integer, String, create_engine + from sqlalchemy.orm import scoped_session, sessionmaker + + Base = declarative_base() + DBSession = scoped_session(sessionmaker()) + engine = None + + + class Customer(Base): + __tablename__ = "customer" + id = Column(Integer, primary_key=True) + name = Column(String(255)) + + + def init_sqlalchemy(dbname='sqlite:///sqlalchemy.db'): + global engine + engine = create_engine(dbname, echo=False) + DBSession.remove() + DBSession.configure(bind=engine, autoflush=False, expire_on_commit=False) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + + def test_sqlalchemy_orm(n=100000): + init_sqlalchemy() + t0 = time.time() + for i in xrange(n): + customer = Customer() + customer.name = 'NAME ' + str(i) + DBSession.add(customer) + if i % 1000 == 0: + DBSession.flush() + DBSession.commit() + print( + "SQLAlchemy ORM: Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + + + def test_sqlalchemy_orm_pk_given(n=100000): + init_sqlalchemy() + t0 = time.time() + for i in xrange(n): + customer = Customer(id=i+1, name="NAME " + str(i)) + DBSession.add(customer) + if i % 1000 == 0: + DBSession.flush() + DBSession.commit() + print( + "SQLAlchemy ORM pk given: Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + + + def test_sqlalchemy_orm_bulk_insert(n=100000): + init_sqlalchemy() + t0 = time.time() + n1 = n + while n1 > 0: + n1 = n1 - 10000 + DBSession.bulk_insert_mappings( + Customer, + [ + dict(name="NAME " + str(i)) + for i in xrange(min(10000, n1)) + ] + ) + DBSession.commit() + print( + "SQLAlchemy ORM bulk_save_objects(): Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + + + def test_sqlalchemy_core(n=100000): + init_sqlalchemy() + t0 = time.time() + engine.execute( + Customer.__table__.insert(), + [{"name": 'NAME ' + str(i)} for i in xrange(n)] + ) + print( + "SQLAlchemy Core: Total time for " + str(n) + + " records " + str(time.time() - t0) + " secs") + + + def init_sqlite3(dbname): + conn = sqlite3.connect(dbname) + c = conn.cursor() + c.execute("DROP TABLE IF EXISTS customer") + c.execute( + "CREATE TABLE customer (id INTEGER NOT NULL, " + "name VARCHAR(255), PRIMARY KEY(id))") + conn.commit() + return conn + + + def test_sqlite3(n=100000, dbname='sqlite3.db'): + conn = init_sqlite3(dbname) + c = conn.cursor() + t0 = time.time() + for i in xrange(n): + row = ('NAME ' + str(i),) + c.execute("INSERT INTO customer (name) VALUES (?)", row) + conn.commit() + print( + "sqlite3: Total time for " + str(n) + + " records " + str(time.time() - t0) + " sec") + + if __name__ == '__main__': + test_sqlalchemy_orm(100000) + test_sqlalchemy_orm_pk_given(100000) + test_sqlalchemy_orm_bulk_insert(100000) + test_sqlalchemy_core(100000) + test_sqlite3(100000) + diff --git a/doc/build/faq/sessions.rst b/doc/build/faq/sessions.rst new file mode 100644 index 000000000..300b4bdbc --- /dev/null +++ b/doc/build/faq/sessions.rst @@ -0,0 +1,363 @@ +Sessions / Queries +=================== + +.. contents:: + :local: + :class: faq + :backlinks: none + + +"This Session's transaction has been rolled back due to a previous exception during flush." (or similar) +--------------------------------------------------------------------------------------------------------- + +This is an error that occurs when a :meth:`.Session.flush` raises an exception, rolls back +the transaction, but further commands upon the `Session` are called without an +explicit call to :meth:`.Session.rollback` or :meth:`.Session.close`. + +It usually corresponds to an application that catches an exception +upon :meth:`.Session.flush` or :meth:`.Session.commit` and +does not properly handle the exception. For example:: + + from sqlalchemy import create_engine, Column, Integer + from sqlalchemy.orm import sessionmaker + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base(create_engine('sqlite://')) + + class Foo(Base): + __tablename__ = 'foo' + id = Column(Integer, primary_key=True) + + Base.metadata.create_all() + + session = sessionmaker()() + + # constraint violation + session.add_all([Foo(id=1), Foo(id=1)]) + + try: + session.commit() + except: + # ignore error + pass + + # continue using session without rolling back + session.commit() + + +The usage of the :class:`.Session` should fit within a structure similar to this:: + + try: + + session.commit() + except: + session.rollback() + raise + finally: + session.close() # optional, depends on use case + +Many things can cause a failure within the try/except besides flushes. You +should always have some kind of "framing" of your session operations so that +connection and transaction resources have a definitive boundary, otherwise +your application doesn't really have its usage of resources under control. +This is not to say that you need to put try/except blocks all throughout your +application - on the contrary, this would be a terrible idea. You should +architect your application such that there is one (or few) point(s) of +"framing" around session operations. + +For a detailed discussion on how to organize usage of the :class:`.Session`, +please see :ref:`session_faq_whentocreate`. + +But why does flush() insist on issuing a ROLLBACK? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It would be great if :meth:`.Session.flush` could partially complete and then not roll +back, however this is beyond its current capabilities since its internal +bookkeeping would have to be modified such that it can be halted at any time +and be exactly consistent with what's been flushed to the database. While this +is theoretically possible, the usefulness of the enhancement is greatly +decreased by the fact that many database operations require a ROLLBACK in any +case. Postgres in particular has operations which, once failed, the +transaction is not allowed to continue:: + + test=> create table foo(id integer primary key); + NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "foo_pkey" for table "foo" + CREATE TABLE + test=> begin; + BEGIN + test=> insert into foo values(1); + INSERT 0 1 + test=> commit; + COMMIT + test=> begin; + BEGIN + test=> insert into foo values(1); + ERROR: duplicate key value violates unique constraint "foo_pkey" + test=> insert into foo values(2); + ERROR: current transaction is aborted, commands ignored until end of transaction block + +What SQLAlchemy offers that solves both issues is support of SAVEPOINT, via +:meth:`.Session.begin_nested`. Using :meth:`.Session.begin_nested`, you can frame an operation that may +potentially fail within a transaction, and then "roll back" to the point +before its failure while maintaining the enclosing transaction. + +But why isn't the one automatic call to ROLLBACK enough? Why must I ROLLBACK again? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is again a matter of the :class:`.Session` providing a consistent interface and +refusing to guess about what context its being used. For example, the +:class:`.Session` supports "framing" above within multiple levels. Such as, suppose +you had a decorator ``@with_session()``, which did this:: + + def with_session(fn): + def go(*args, **kw): + session.begin(subtransactions=True) + try: + ret = fn(*args, **kw) + session.commit() + return ret + except: + session.rollback() + raise + return go + +The above decorator begins a transaction if one does not exist already, and +then commits it, if it were the creator. The "subtransactions" flag means that +if :meth:`.Session.begin` were already called by an enclosing function, nothing happens +except a counter is incremented - this counter is decremented when :meth:`.Session.commit` +is called and only when it goes back to zero does the actual COMMIT happen. It +allows this usage pattern:: + + @with_session + def one(): + # do stuff + two() + + + @with_session + def two(): + # etc. + + one() + + two() + +``one()`` can call ``two()``, or ``two()`` can be called by itself, and the +``@with_session`` decorator ensures the appropriate "framing" - the transaction +boundaries stay on the outermost call level. As you can see, if ``two()`` calls +``flush()`` which throws an exception and then issues a ``rollback()``, there will +*always* be a second ``rollback()`` performed by the decorator, and possibly a +third corresponding to two levels of decorator. If the ``flush()`` pushed the +``rollback()`` all the way out to the top of the stack, and then we said that +all remaining ``rollback()`` calls are moot, there is some silent behavior going +on there. A poorly written enclosing method might suppress the exception, and +then call ``commit()`` assuming nothing is wrong, and then you have a silent +failure condition. The main reason people get this error in fact is because +they didn't write clean "framing" code and they would have had other problems +down the road. + +If you think the above use case is a little exotic, the same kind of thing +comes into play if you want to SAVEPOINT- you might call ``begin_nested()`` +several times, and the ``commit()``/``rollback()`` calls each resolve the most +recent ``begin_nested()``. The meaning of ``rollback()`` or ``commit()`` is +dependent upon which enclosing block it is called, and you might have any +sequence of ``rollback()``/``commit()`` in any order, and its the level of nesting +that determines their behavior. + +In both of the above cases, if ``flush()`` broke the nesting of transaction +blocks, the behavior is, depending on scenario, anywhere from "magic" to +silent failure to blatant interruption of code flow. + +``flush()`` makes its own "subtransaction", so that a transaction is started up +regardless of the external transactional state, and when complete it calls +``commit()``, or ``rollback()`` upon failure - but that ``rollback()`` corresponds +to its own subtransaction - it doesn't want to guess how you'd like to handle +the external "framing" of the transaction, which could be nested many levels +with any combination of subtransactions and real SAVEPOINTs. The job of +starting/ending the "frame" is kept consistently with the code external to the +``flush()``, and we made a decision that this was the most consistent approach. + + + +How do I make a Query that always adds a certain filter to every query? +------------------------------------------------------------------------------------------------ + +See the recipe at `PreFilteredQuery `_. + +I've created a mapping against an Outer Join, and while the query returns rows, no objects are returned. Why not? +------------------------------------------------------------------------------------------------------------------ + +Rows returned by an outer join may contain NULL for part of the primary key, +as the primary key is the composite of both tables. The :class:`.Query` object ignores incoming rows +that don't have an acceptable primary key. Based on the setting of the ``allow_partial_pks`` +flag on :func:`.mapper`, a primary key is accepted if the value has at least one non-NULL +value, or alternatively if the value has no NULL values. See ``allow_partial_pks`` +at :func:`.mapper`. + + +I'm using ``joinedload()`` or ``lazy=False`` to create a JOIN/OUTER JOIN and SQLAlchemy is not constructing the correct query when I try to add a WHERE, ORDER BY, LIMIT, etc. (which relies upon the (OUTER) JOIN) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +The joins generated by joined eager loading are only used to fully load related +collections, and are designed to have no impact on the primary results of the query. +Since they are anonymously aliased, they cannot be referenced directly. + +For detail on this beahvior, see :doc:`orm/loading`. + +Query has no ``__len__()``, why not? +------------------------------------ + +The Python ``__len__()`` magic method applied to an object allows the ``len()`` +builtin to be used to determine the length of the collection. It's intuitive +that a SQL query object would link ``__len__()`` to the :meth:`.Query.count` +method, which emits a `SELECT COUNT`. The reason this is not possible is +because evaluating the query as a list would incur two SQL calls instead of +one:: + + class Iterates(object): + def __len__(self): + print "LEN!" + return 5 + + def __iter__(self): + print "ITER!" + return iter([1, 2, 3, 4, 5]) + + list(Iterates()) + +output:: + + ITER! + LEN! + +How Do I use Textual SQL with ORM Queries? +------------------------------------------- + +See: + +* :ref:`orm_tutorial_literal_sql` - Ad-hoc textual blocks with :class:`.Query` + +* :ref:`session_sql_expressions` - Using :class:`.Session` with textual SQL directly. + +I'm calling ``Session.delete(myobject)`` and it isn't removed from the parent collection! +------------------------------------------------------------------------------------------ + +See :ref:`session_deleting_from_collections` for a description of this behavior. + +why isn't my ``__init__()`` called when I load objects? +------------------------------------------------------- + +See :ref:`mapping_constructors` for a description of this behavior. + +how do I use ON DELETE CASCADE with SA's ORM? +---------------------------------------------- + +SQLAlchemy will always issue UPDATE or DELETE statements for dependent +rows which are currently loaded in the :class:`.Session`. For rows which +are not loaded, it will by default issue SELECT statements to load +those rows and udpate/delete those as well; in other words it assumes +there is no ON DELETE CASCADE configured. +To configure SQLAlchemy to cooperate with ON DELETE CASCADE, see +:ref:`passive_deletes`. + +I set the "foo_id" attribute on my instance to "7", but the "foo" attribute is still ``None`` - shouldn't it have loaded Foo with id #7? +---------------------------------------------------------------------------------------------------------------------------------------------------- + +The ORM is not constructed in such a way as to support +immediate population of relationships driven from foreign +key attribute changes - instead, it is designed to work the +other way around - foreign key attributes are handled by the +ORM behind the scenes, the end user sets up object +relationships naturally. Therefore, the recommended way to +set ``o.foo`` is to do just that - set it!:: + + foo = Session.query(Foo).get(7) + o.foo = foo + Session.commit() + +Manipulation of foreign key attributes is of course entirely legal. However, +setting a foreign-key attribute to a new value currently does not trigger +an "expire" event of the :func:`.relationship` in which it's involved. This means +that for the following sequence:: + + o = Session.query(SomeClass).first() + assert o.foo is None # accessing an un-set attribute sets it to None + o.foo_id = 7 + +``o.foo`` is initialized to ``None`` when we first accessed it. Setting +``o.foo_id = 7`` will have the value of "7" as pending, but no flush +has occurred - so ``o.foo`` is still ``None``:: + + # attribute is already set to None, has not been + # reconciled with o.foo_id = 7 yet + assert o.foo is None + +For ``o.foo`` to load based on the foreign key mutation is usually achieved +naturally after the commit, which both flushes the new foreign key value +and expires all state:: + + Session.commit() # expires all attributes + + foo_7 = Session.query(Foo).get(7) + + assert o.foo is foo_7 # o.foo lazyloads on access + +A more minimal operation is to expire the attribute individually - this can +be performed for any :term:`persistent` object using :meth:`.Session.expire`:: + + o = Session.query(SomeClass).first() + o.foo_id = 7 + Session.expire(o, ['foo']) # object must be persistent for this + + foo_7 = Session.query(Foo).get(7) + + assert o.foo is foo_7 # o.foo lazyloads on access + +Note that if the object is not persistent but present in the :class:`.Session`, +it's known as :term:`pending`. This means the row for the object has not been +INSERTed into the database yet. For such an object, setting ``foo_id`` does not +have meaning until the row is inserted; otherwise there is no row yet:: + + new_obj = SomeClass() + new_obj.foo_id = 7 + + Session.add(new_obj) + + # accessing an un-set attribute sets it to None + assert new_obj.foo is None + + Session.flush() # emits INSERT + + # expire this because we already set .foo to None + Session.expire(o, ['foo']) + + assert new_obj.foo is foo_7 # now it loads + + +.. topic:: Attribute loading for non-persistent objects + + One variant on the "pending" behavior above is if we use the flag + ``load_on_pending`` on :func:`.relationship`. When this flag is set, the + lazy loader will emit for ``new_obj.foo`` before the INSERT proceeds; another + variant of this is to use the :meth:`.Session.enable_relationship_loading` + method, which can "attach" an object to a :class:`.Session` in such a way that + many-to-one relationships load as according to foreign key attributes + regardless of the object being in any particular state. + Both techniques are **not recommended for general use**; they were added to suit + specific programming scenarios encountered by users which involve the repurposing + of the ORM's usual object states. + +The recipe `ExpireRelationshipOnFKChange `_ features an example using SQLAlchemy events +in order to coordinate the setting of foreign key attributes with many-to-one +relationships. + +Is there a way to automagically have only unique keywords (or other kinds of objects) without doing a query for the keyword and getting a reference to the row containing that keyword? +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +When people read the many-to-many example in the docs, they get hit with the +fact that if you create the same ``Keyword`` twice, it gets put in the DB twice. +Which is somewhat inconvenient. + +This `UniqueObject `_ recipe was created to address this issue. + + diff --git a/doc/build/faq/sqlexpressions.rst b/doc/build/faq/sqlexpressions.rst new file mode 100644 index 000000000..c3504218b --- /dev/null +++ b/doc/build/faq/sqlexpressions.rst @@ -0,0 +1,140 @@ +SQL Expressions +================= + +.. contents:: + :local: + :class: faq + :backlinks: none + +.. _faq_sql_expression_string: + +How do I render SQL expressions as strings, possibly with bound parameters inlined? +------------------------------------------------------------------------------------ + +The "stringification" of a SQLAlchemy statement or Query in the vast majority +of cases is as simple as:: + + print(str(statement)) + +this applies both to an ORM :class:`~.orm.query.Query` as well as any :func:`.select` or other +statement. Additionally, to get the statement as compiled to a +specific dialect or engine, if the statement itself is not already +bound to one you can pass this in to :meth:`.ClauseElement.compile`:: + + print(statement.compile(someengine)) + +or without an :class:`.Engine`:: + + from sqlalchemy.dialects import postgresql + print(statement.compile(dialect=postgresql.dialect())) + +When given an ORM :class:`~.orm.query.Query` object, in order to get at the +:meth:`.ClauseElement.compile` +method we only need access the :attr:`~.orm.query.Query.statement` +accessor first:: + + statement = query.statement + print(statement.compile(someengine)) + +The above forms will render the SQL statement as it is passed to the Python +:term:`DBAPI`, which includes that bound parameters are not rendered inline. +SQLAlchemy normally does not stringify bound parameters, as this is handled +appropriately by the Python DBAPI, not to mention bypassing bound +parameters is probably the most widely exploited security hole in +modern web applications. SQLAlchemy has limited ability to do this +stringification in certain circumstances such as that of emitting DDL. +In order to access this functionality one can use the ``literal_binds`` +flag, passed to ``compile_kwargs``:: + + from sqlalchemy.sql import table, column, select + + t = table('t', column('x')) + + s = select([t]).where(t.c.x == 5) + + print(s.compile(compile_kwargs={"literal_binds": True})) + +the above approach has the caveats that it is only supported for basic +types, such as ints and strings, and furthermore if a :func:`.bindparam` +without a pre-set value is used directly, it won't be able to +stringify that either. + +To support inline literal rendering for types not supported, implement +a :class:`.TypeDecorator` for the target type which includes a +:meth:`.TypeDecorator.process_literal_param` method:: + + from sqlalchemy import TypeDecorator, Integer + + + class MyFancyType(TypeDecorator): + impl = Integer + + def process_literal_param(self, value, dialect): + return "my_fancy_formatting(%s)" % value + + from sqlalchemy import Table, Column, MetaData + + tab = Table('mytable', MetaData(), Column('x', MyFancyType())) + + print( + tab.select().where(tab.c.x > 5).compile( + compile_kwargs={"literal_binds": True}) + ) + +producing output like:: + + SELECT mytable.x + FROM mytable + WHERE mytable.x > my_fancy_formatting(5) + + +Why does ``.col.in_([])`` Produce ``col != col``? Why not ``1=0``? +------------------------------------------------------------------- + +A little introduction to the issue. The IN operator in SQL, given a list of +elements to compare against a column, generally does not accept an empty list, +that is while it is valid to say:: + + column IN (1, 2, 3) + +it's not valid to say:: + + column IN () + +SQLAlchemy's :meth:`.Operators.in_` operator, when given an empty list, produces this +expression:: + + column != column + +As of version 0.6, it also produces a warning stating that a less efficient +comparison operation will be rendered. This expression is the only one that is +both database agnostic and produces correct results. + +For example, the naive approach of "just evaluate to false, by comparing 1=0 +or 1!=1", does not handle nulls properly. An expression like:: + + NOT column != column + +will not return a row when "column" is null, but an expression which does not +take the column into account:: + + NOT 1=0 + +will. + +Closer to the mark is the following CASE expression:: + + CASE WHEN column IS NOT NULL THEN 1=0 ELSE NULL END + +We don't use this expression due to its verbosity, and its also not +typically accepted by Oracle within a WHERE clause - depending +on how you phrase it, you'll either get "ORA-00905: missing keyword" or +"ORA-00920: invalid relational operator". It's also still less efficient than +just rendering SQL without the clause altogether (or not issuing the SQL at +all, if the statement is just a simple search). + +The best approach therefore is to avoid the usage of IN given an argument list +of zero length. Instead, don't emit the Query in the first place, if no rows +should be returned. The warning is best promoted to a full error condition +using the Python warnings filter (see http://docs.python.org/library/warnings.html). + diff --git a/doc/build/index.rst b/doc/build/index.rst index 205a5c12b..8b60ef9b9 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -13,7 +13,7 @@ A high level view and getting set up. :doc:`Overview ` | :ref:`Installation Guide ` | -:doc:`Frequently Asked Questions ` | +:doc:`Frequently Asked Questions ` | :doc:`Migration from 0.9 ` | :doc:`Glossary ` | :doc:`Changelog catalog ` @@ -32,32 +32,23 @@ of Python objects, proceed first to the tutorial. * **ORM Configuration:** :doc:`Mapper Configuration ` | :doc:`Relationship Configuration ` | - :doc:`Inheritance Mapping ` | - :doc:`Advanced Collection Configuration ` * **Configuration Extensions:** - :doc:`Declarative Extension ` | + :doc:`Declarative Extension ` | :doc:`Association Proxy ` | :doc:`Hybrid Attributes ` | :doc:`Automap ` | - :doc:`Mutable Scalars ` | - :doc:`Ordered List ` + :doc:`Mutable Scalars ` * **ORM Usage:** :doc:`Session Usage and Guidelines ` | - :doc:`Query API reference ` | - :doc:`Relationship Loading Techniques ` + :doc:`Loading Objects ` * **Extending the ORM:** - :doc:`ORM Event Interfaces ` | - :doc:`Internals API ` + :doc:`ORM Events and Internals ` * **Other:** - :doc:`Introduction to Examples ` | - :doc:`Deprecated Event Interfaces ` | - :doc:`ORM Exceptions ` | - :doc:`Horizontal Sharding ` | - :doc:`Alternate Instrumentation ` + :doc:`Introduction to Examples ` SQLAlchemy Core =============== @@ -78,6 +69,7 @@ are documented here. In contrast to the ORM's domain-centric mode of usage, the :doc:`Connection Pooling ` * **Schema Definition:** + :doc:`Overview ` | :ref:`Tables and Columns ` | :ref:`Database Introspection (Reflection) ` | :ref:`Insert/Update Defaults ` | @@ -86,23 +78,15 @@ are documented here. In contrast to the ORM's domain-centric mode of usage, the * **Datatypes:** :ref:`Overview ` | - :ref:`Generic Types ` | - :ref:`SQL Standard Types ` | - :ref:`Vendor Specific Types ` | :ref:`Building Custom Types ` | - :ref:`Defining New Operators ` | :ref:`API ` -* **Extending the Core:** - :doc:`SQLAlchemy Events ` | +* **Core Basics:** + :doc:`Overview ` | + :doc:`Runtime Inspection API ` | + :doc:`Event System ` | :doc:`Core Event Interfaces ` | :doc:`Creating Custom SQL Constructs ` | - :doc:`Internals API ` - -* **Other:** - :doc:`Runtime Inspection API ` | - :doc:`core/interfaces` | - :doc:`core/exceptions` Dialect Documentation diff --git a/doc/build/orm/backref.rst b/doc/build/orm/backref.rst new file mode 100644 index 000000000..16cfe5606 --- /dev/null +++ b/doc/build/orm/backref.rst @@ -0,0 +1,273 @@ +.. _relationships_backref: + +Linking Relationships with Backref +---------------------------------- + +The :paramref:`~.relationship.backref` keyword argument was first introduced in :ref:`ormtutorial_toplevel`, and has been +mentioned throughout many of the examples here. What does it actually do ? Let's start +with the canonical ``User`` and ``Address`` scenario:: + + from sqlalchemy import Integer, ForeignKey, String, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + name = Column(String) + + addresses = relationship("Address", backref="user") + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + email = Column(String) + user_id = Column(Integer, ForeignKey('user.id')) + +The above configuration establishes a collection of ``Address`` objects on ``User`` called +``User.addresses``. It also establishes a ``.user`` attribute on ``Address`` which will +refer to the parent ``User`` object. + +In fact, the :paramref:`~.relationship.backref` keyword is only a common shortcut for placing a second +:func:`.relationship` onto the ``Address`` mapping, including the establishment +of an event listener on both sides which will mirror attribute operations +in both directions. The above configuration is equivalent to:: + + from sqlalchemy import Integer, ForeignKey, String, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + name = Column(String) + + addresses = relationship("Address", back_populates="user") + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + email = Column(String) + user_id = Column(Integer, ForeignKey('user.id')) + + user = relationship("User", back_populates="addresses") + +Above, we add a ``.user`` relationship to ``Address`` explicitly. On +both relationships, the :paramref:`~.relationship.back_populates` directive tells each relationship +about the other one, indicating that they should establish "bidirectional" +behavior between each other. The primary effect of this configuration +is that the relationship adds event handlers to both attributes +which have the behavior of "when an append or set event occurs here, set ourselves +onto the incoming attribute using this particular attribute name". +The behavior is illustrated as follows. Start with a ``User`` and an ``Address`` +instance. The ``.addresses`` collection is empty, and the ``.user`` attribute +is ``None``:: + + >>> u1 = User() + >>> a1 = Address() + >>> u1.addresses + [] + >>> print a1.user + None + +However, once the ``Address`` is appended to the ``u1.addresses`` collection, +both the collection and the scalar attribute have been populated:: + + >>> u1.addresses.append(a1) + >>> u1.addresses + [<__main__.Address object at 0x12a6ed0>] + >>> a1.user + <__main__.User object at 0x12a6590> + +This behavior of course works in reverse for removal operations as well, as well +as for equivalent operations on both sides. Such as +when ``.user`` is set again to ``None``, the ``Address`` object is removed +from the reverse collection:: + + >>> a1.user = None + >>> u1.addresses + [] + +The manipulation of the ``.addresses`` collection and the ``.user`` attribute +occurs entirely in Python without any interaction with the SQL database. +Without this behavior, the proper state would be apparent on both sides once the +data has been flushed to the database, and later reloaded after a commit or +expiration operation occurs. The :paramref:`~.relationship.backref`/:paramref:`~.relationship.back_populates` behavior has the advantage +that common bidirectional operations can reflect the correct state without requiring +a database round trip. + +Remember, when the :paramref:`~.relationship.backref` keyword is used on a single relationship, it's +exactly the same as if the above two relationships were created individually +using :paramref:`~.relationship.back_populates` on each. + +Backref Arguments +~~~~~~~~~~~~~~~~~~ + +We've established that the :paramref:`~.relationship.backref` keyword is merely a shortcut for building +two individual :func:`.relationship` constructs that refer to each other. Part of +the behavior of this shortcut is that certain configurational arguments applied to +the :func:`.relationship` +will also be applied to the other direction - namely those arguments that describe +the relationship at a schema level, and are unlikely to be different in the reverse +direction. The usual case +here is a many-to-many :func:`.relationship` that has a :paramref:`~.relationship.secondary` argument, +or a one-to-many or many-to-one which has a :paramref:`~.relationship.primaryjoin` argument (the +:paramref:`~.relationship.primaryjoin` argument is discussed in :ref:`relationship_primaryjoin`). Such +as if we limited the list of ``Address`` objects to those which start with "tony":: + + from sqlalchemy import Integer, ForeignKey, String, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + name = Column(String) + + addresses = relationship("Address", + primaryjoin="and_(User.id==Address.user_id, " + "Address.email.startswith('tony'))", + backref="user") + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + email = Column(String) + user_id = Column(Integer, ForeignKey('user.id')) + +We can observe, by inspecting the resulting property, that both sides +of the relationship have this join condition applied:: + + >>> print User.addresses.property.primaryjoin + "user".id = address.user_id AND address.email LIKE :email_1 || '%%' + >>> + >>> print Address.user.property.primaryjoin + "user".id = address.user_id AND address.email LIKE :email_1 || '%%' + >>> + +This reuse of arguments should pretty much do the "right thing" - it +uses only arguments that are applicable, and in the case of a many-to- +many relationship, will reverse the usage of +:paramref:`~.relationship.primaryjoin` and +:paramref:`~.relationship.secondaryjoin` to correspond to the other +direction (see the example in :ref:`self_referential_many_to_many` for +this). + +It's very often the case however that we'd like to specify arguments +that are specific to just the side where we happened to place the +"backref". This includes :func:`.relationship` arguments like +:paramref:`~.relationship.lazy`, +:paramref:`~.relationship.remote_side`, +:paramref:`~.relationship.cascade` and +:paramref:`~.relationship.cascade_backrefs`. For this case we use +the :func:`.backref` function in place of a string:: + + # + from sqlalchemy.orm import backref + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + name = Column(String) + + addresses = relationship("Address", + backref=backref("user", lazy="joined")) + +Where above, we placed a ``lazy="joined"`` directive only on the ``Address.user`` +side, indicating that when a query against ``Address`` is made, a join to the ``User`` +entity should be made automatically which will populate the ``.user`` attribute of each +returned ``Address``. The :func:`.backref` function formatted the arguments we gave +it into a form that is interpreted by the receiving :func:`.relationship` as additional +arguments to be applied to the new relationship it creates. + +One Way Backrefs +~~~~~~~~~~~~~~~~~ + +An unusual case is that of the "one way backref". This is where the +"back-populating" behavior of the backref is only desirable in one +direction. An example of this is a collection which contains a +filtering :paramref:`~.relationship.primaryjoin` condition. We'd +like to append items to this collection as needed, and have them +populate the "parent" object on the incoming object. However, we'd +also like to have items that are not part of the collection, but still +have the same "parent" association - these items should never be in +the collection. + +Taking our previous example, where we established a +:paramref:`~.relationship.primaryjoin` that limited the collection +only to ``Address`` objects whose email address started with the word +``tony``, the usual backref behavior is that all items populate in +both directions. We wouldn't want this behavior for a case like the +following:: + + >>> u1 = User() + >>> a1 = Address(email='mary') + >>> a1.user = u1 + >>> u1.addresses + [<__main__.Address object at 0x1411910>] + +Above, the ``Address`` object that doesn't match the criterion of "starts with 'tony'" +is present in the ``addresses`` collection of ``u1``. After these objects are flushed, +the transaction committed and their attributes expired for a re-load, the ``addresses`` +collection will hit the database on next access and no longer have this ``Address`` object +present, due to the filtering condition. But we can do away with this unwanted side +of the "backref" behavior on the Python side by using two separate :func:`.relationship` constructs, +placing :paramref:`~.relationship.back_populates` only on one side:: + + from sqlalchemy import Integer, ForeignKey, String, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + name = Column(String) + addresses = relationship("Address", + primaryjoin="and_(User.id==Address.user_id, " + "Address.email.startswith('tony'))", + back_populates="user") + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + email = Column(String) + user_id = Column(Integer, ForeignKey('user.id')) + user = relationship("User") + +With the above scenario, appending an ``Address`` object to the ``.addresses`` +collection of a ``User`` will always establish the ``.user`` attribute on that +``Address``:: + + >>> u1 = User() + >>> a1 = Address(email='tony') + >>> u1.addresses.append(a1) + >>> a1.user + <__main__.User object at 0x1411850> + +However, applying a ``User`` to the ``.user`` attribute of an ``Address``, +will not append the ``Address`` object to the collection:: + + >>> a2 = Address(email='mary') + >>> a2.user = u1 + >>> a2 in u1.addresses + False + +Of course, we've disabled some of the usefulness of +:paramref:`~.relationship.backref` here, in that when we do append an +``Address`` that corresponds to the criteria of +``email.startswith('tony')``, it won't show up in the +``User.addresses`` collection until the session is flushed, and the +attributes reloaded after a commit or expire operation. While we +could consider an attribute event that checks this criterion in +Python, this starts to cross the line of duplicating too much SQL +behavior in Python. The backref behavior itself is only a slight +transgression of this philosophy - SQLAlchemy tries to keep these to a +minimum overall. diff --git a/doc/build/orm/basic_relationships.rst b/doc/build/orm/basic_relationships.rst new file mode 100644 index 000000000..9a7ad4fa2 --- /dev/null +++ b/doc/build/orm/basic_relationships.rst @@ -0,0 +1,313 @@ +.. _relationship_patterns: + +Basic Relationship Patterns +---------------------------- + +A quick walkthrough of the basic relational patterns. + +The imports used for each of the following sections is as follows:: + + from sqlalchemy import Table, Column, Integer, ForeignKey + from sqlalchemy.orm import relationship, backref + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + +One To Many +~~~~~~~~~~~~ + +A one to many relationship places a foreign key on the child table referencing +the parent. :func:`.relationship` is then specified on the parent, as referencing +a collection of items represented by the child:: + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + children = relationship("Child") + + class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('parent.id')) + +To establish a bidirectional relationship in one-to-many, where the "reverse" +side is a many to one, specify the :paramref:`~.relationship.backref` option:: + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + children = relationship("Child", backref="parent") + + class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('parent.id')) + +``Child`` will get a ``parent`` attribute with many-to-one semantics. + +Many To One +~~~~~~~~~~~~ + +Many to one places a foreign key in the parent table referencing the child. +:func:`.relationship` is declared on the parent, where a new scalar-holding +attribute will be created:: + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + child_id = Column(Integer, ForeignKey('child.id')) + child = relationship("Child") + + class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + +Bidirectional behavior is achieved by setting +:paramref:`~.relationship.backref` to the value ``"parents"``, which +will place a one-to-many collection on the ``Child`` class:: + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + child_id = Column(Integer, ForeignKey('child.id')) + child = relationship("Child", backref="parents") + +.. _relationships_one_to_one: + +One To One +~~~~~~~~~~~ + +One To One is essentially a bidirectional relationship with a scalar +attribute on both sides. To achieve this, the :paramref:`~.relationship.uselist` flag indicates +the placement of a scalar attribute instead of a collection on the "many" side +of the relationship. To convert one-to-many into one-to-one:: + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + child = relationship("Child", uselist=False, backref="parent") + + class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('parent.id')) + +Or to turn a one-to-many backref into one-to-one, use the :func:`.backref` function +to provide arguments for the reverse side:: + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + child_id = Column(Integer, ForeignKey('child.id')) + child = relationship("Child", backref=backref("parent", uselist=False)) + + class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + +.. _relationships_many_to_many: + +Many To Many +~~~~~~~~~~~~~ + +Many to Many adds an association table between two classes. The association +table is indicated by the :paramref:`~.relationship.secondary` argument to +:func:`.relationship`. Usually, the :class:`.Table` uses the :class:`.MetaData` +object associated with the declarative base class, so that the :class:`.ForeignKey` +directives can locate the remote tables with which to link:: + + association_table = Table('association', Base.metadata, + Column('left_id', Integer, ForeignKey('left.id')), + Column('right_id', Integer, ForeignKey('right.id')) + ) + + class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + children = relationship("Child", + secondary=association_table) + + class Child(Base): + __tablename__ = 'right' + id = Column(Integer, primary_key=True) + +For a bidirectional relationship, both sides of the relationship contain a +collection. The :paramref:`~.relationship.backref` keyword will automatically use +the same :paramref:`~.relationship.secondary` argument for the reverse relationship:: + + association_table = Table('association', Base.metadata, + Column('left_id', Integer, ForeignKey('left.id')), + Column('right_id', Integer, ForeignKey('right.id')) + ) + + class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + children = relationship("Child", + secondary=association_table, + backref="parents") + + class Child(Base): + __tablename__ = 'right' + id = Column(Integer, primary_key=True) + +The :paramref:`~.relationship.secondary` argument of :func:`.relationship` also accepts a callable +that returns the ultimate argument, which is evaluated only when mappers are +first used. Using this, we can define the ``association_table`` at a later +point, as long as it's available to the callable after all module initialization +is complete:: + + class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + children = relationship("Child", + secondary=lambda: association_table, + backref="parents") + +With the declarative extension in use, the traditional "string name of the table" +is accepted as well, matching the name of the table as stored in ``Base.metadata.tables``:: + + class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + children = relationship("Child", + secondary="association", + backref="parents") + +.. _relationships_many_to_many_deletion: + +Deleting Rows from the Many to Many Table +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A behavior which is unique to the :paramref:`~.relationship.secondary` argument to :func:`.relationship` +is that the :class:`.Table` which is specified here is automatically subject +to INSERT and DELETE statements, as objects are added or removed from the collection. +There is **no need to delete from this table manually**. The act of removing a +record from the collection will have the effect of the row being deleted on flush:: + + # row will be deleted from the "secondary" table + # automatically + myparent.children.remove(somechild) + +A question which often arises is how the row in the "secondary" table can be deleted +when the child object is handed directly to :meth:`.Session.delete`:: + + session.delete(somechild) + +There are several possibilities here: + +* If there is a :func:`.relationship` from ``Parent`` to ``Child``, but there is + **not** a reverse-relationship that links a particular ``Child`` to each ``Parent``, + SQLAlchemy will not have any awareness that when deleting this particular + ``Child`` object, it needs to maintain the "secondary" table that links it to + the ``Parent``. No delete of the "secondary" table will occur. +* If there is a relationship that links a particular ``Child`` to each ``Parent``, + suppose it's called ``Child.parents``, SQLAlchemy by default will load in + the ``Child.parents`` collection to locate all ``Parent`` objects, and remove + each row from the "secondary" table which establishes this link. Note that + this relationship does not need to be bidrectional; SQLAlchemy is strictly + looking at every :func:`.relationship` associated with the ``Child`` object + being deleted. +* A higher performing option here is to use ON DELETE CASCADE directives + with the foreign keys used by the database. Assuming the database supports + this feature, the database itself can be made to automatically delete rows in the + "secondary" table as referencing rows in "child" are deleted. SQLAlchemy + can be instructed to forego actively loading in the ``Child.parents`` + collection in this case using the :paramref:`~.relationship.passive_deletes` + directive on :func:`.relationship`; see :ref:`passive_deletes` for more details + on this. + +Note again, these behaviors are *only* relevant to the :paramref:`~.relationship.secondary` option +used with :func:`.relationship`. If dealing with association tables that +are mapped explicitly and are *not* present in the :paramref:`~.relationship.secondary` option +of a relevant :func:`.relationship`, cascade rules can be used instead +to automatically delete entities in reaction to a related entity being +deleted - see :ref:`unitofwork_cascades` for information on this feature. + + +.. _association_pattern: + +Association Object +~~~~~~~~~~~~~~~~~~ + +The association object pattern is a variant on many-to-many: it's used +when your association table contains additional columns beyond those +which are foreign keys to the left and right tables. Instead of using +the :paramref:`~.relationship.secondary` argument, you map a new class +directly to the association table. The left side of the relationship +references the association object via one-to-many, and the association +class references the right side via many-to-one. Below we illustrate +an association table mapped to the ``Association`` class which +includes a column called ``extra_data``, which is a string value that +is stored along with each association between ``Parent`` and +``Child``:: + + class Association(Base): + __tablename__ = 'association' + left_id = Column(Integer, ForeignKey('left.id'), primary_key=True) + right_id = Column(Integer, ForeignKey('right.id'), primary_key=True) + extra_data = Column(String(50)) + child = relationship("Child") + + class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + children = relationship("Association") + + class Child(Base): + __tablename__ = 'right' + id = Column(Integer, primary_key=True) + +The bidirectional version adds backrefs to both relationships:: + + class Association(Base): + __tablename__ = 'association' + left_id = Column(Integer, ForeignKey('left.id'), primary_key=True) + right_id = Column(Integer, ForeignKey('right.id'), primary_key=True) + extra_data = Column(String(50)) + child = relationship("Child", backref="parent_assocs") + + class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + children = relationship("Association", backref="parent") + + class Child(Base): + __tablename__ = 'right' + id = Column(Integer, primary_key=True) + +Working with the association pattern in its direct form requires that child +objects are associated with an association instance before being appended to +the parent; similarly, access from parent to child goes through the +association object:: + + # create parent, append a child via association + p = Parent() + a = Association(extra_data="some data") + a.child = Child() + p.children.append(a) + + # iterate through child objects via association, including association + # attributes + for assoc in p.children: + print assoc.extra_data + print assoc.child + +To enhance the association object pattern such that direct +access to the ``Association`` object is optional, SQLAlchemy +provides the :ref:`associationproxy_toplevel` extension. This +extension allows the configuration of attributes which will +access two "hops" with a single access, one "hop" to the +associated object, and a second to a target attribute. + +.. note:: + + When using the association object pattern, it is advisable that the + association-mapped table not be used as the + :paramref:`~.relationship.secondary` argument on a + :func:`.relationship` elsewhere, unless that :func:`.relationship` + contains the option :paramref:`~.relationship.viewonly` set to + ``True``. SQLAlchemy otherwise may attempt to emit redundant INSERT + and DELETE statements on the same table, if similar state is + detected on the related attribute as well as the associated object. diff --git a/doc/build/orm/cascades.rst b/doc/build/orm/cascades.rst new file mode 100644 index 000000000..f645e6dae --- /dev/null +++ b/doc/build/orm/cascades.rst @@ -0,0 +1,372 @@ +.. _unitofwork_cascades: + +Cascades +======== + +Mappers support the concept of configurable :term:`cascade` behavior on +:func:`~sqlalchemy.orm.relationship` constructs. This refers +to how operations performed on a "parent" object relative to a +particular :class:`.Session` should be propagated to items +referred to by that relationship (e.g. "child" objects), and is +affected by the :paramref:`.relationship.cascade` option. + +The default behavior of cascade is limited to cascades of the +so-called :ref:`cascade_save_update` and :ref:`cascade_merge` settings. +The typical "alternative" setting for cascade is to add +the :ref:`cascade_delete` and :ref:`cascade_delete_orphan` options; +these settings are appropriate for related objects which only exist as +long as they are attached to their parent, and are otherwise deleted. + +Cascade behavior is configured using the by changing the +:paramref:`~.relationship.cascade` option on +:func:`~sqlalchemy.orm.relationship`:: + + class Order(Base): + __tablename__ = 'order' + + items = relationship("Item", cascade="all, delete-orphan") + customer = relationship("User", cascade="save-update") + +To set cascades on a backref, the same flag can be used with the +:func:`~.sqlalchemy.orm.backref` function, which ultimately feeds +its arguments back into :func:`~sqlalchemy.orm.relationship`:: + + class Item(Base): + __tablename__ = 'item' + + order = relationship("Order", + backref=backref("items", cascade="all, delete-orphan") + ) + +.. sidebar:: The Origins of Cascade + + SQLAlchemy's notion of cascading behavior on relationships, + as well as the options to configure them, are primarily derived + from the similar feature in the Hibernate ORM; Hibernate refers + to "cascade" in a few places such as in + `Example: Parent/Child `_. + If cascades are confusing, we'll refer to their conclusion, + stating "The sections we have just covered can be a bit confusing. + However, in practice, it all works out nicely." + +The default value of :paramref:`~.relationship.cascade` is ``save-update, merge``. +The typical alternative setting for this parameter is either +``all`` or more commonly ``all, delete-orphan``. The ``all`` symbol +is a synonym for ``save-update, merge, refresh-expire, expunge, delete``, +and using it in conjunction with ``delete-orphan`` indicates that the child +object should follow along with its parent in all cases, and be deleted once +it is no longer associated with that parent. + +The list of available values which can be specified for +the :paramref:`~.relationship.cascade` parameter are described in the following subsections. + +.. _cascade_save_update: + +save-update +----------- + +``save-update`` cascade indicates that when an object is placed into a +:class:`.Session` via :meth:`.Session.add`, all the objects associated +with it via this :func:`.relationship` should also be added to that +same :class:`.Session`. Suppose we have an object ``user1`` with two +related objects ``address1``, ``address2``:: + + >>> user1 = User() + >>> address1, address2 = Address(), Address() + >>> user1.addresses = [address1, address2] + +If we add ``user1`` to a :class:`.Session`, it will also add +``address1``, ``address2`` implicitly:: + + >>> sess = Session() + >>> sess.add(user1) + >>> address1 in sess + True + +``save-update`` cascade also affects attribute operations for objects +that are already present in a :class:`.Session`. If we add a third +object, ``address3`` to the ``user1.addresses`` collection, it +becomes part of the state of that :class:`.Session`:: + + >>> address3 = Address() + >>> user1.append(address3) + >>> address3 in sess + >>> True + +``save-update`` has the possibly surprising behavior which is that +persistent objects which were *removed* from a collection +or in some cases a scalar attribute +may also be pulled into the :class:`.Session` of a parent object; this is +so that the flush process may handle that related object appropriately. +This case can usually only arise if an object is removed from one :class:`.Session` +and added to another:: + + >>> user1 = sess1.query(User).filter_by(id=1).first() + >>> address1 = user1.addresses[0] + >>> sess1.close() # user1, address1 no longer associated with sess1 + >>> user1.addresses.remove(address1) # address1 no longer associated with user1 + >>> sess2 = Session() + >>> sess2.add(user1) # ... but it still gets added to the new session, + >>> address1 in sess2 # because it's still "pending" for flush + True + +The ``save-update`` cascade is on by default, and is typically taken +for granted; it simplifies code by allowing a single call to +:meth:`.Session.add` to register an entire structure of objects within +that :class:`.Session` at once. While it can be disabled, there +is usually not a need to do so. + +One case where ``save-update`` cascade does sometimes get in the way is in that +it takes place in both directions for bi-directional relationships, e.g. +backrefs, meaning that the association of a child object with a particular parent +can have the effect of the parent object being implicitly associated with that +child object's :class:`.Session`; this pattern, as well as how to modify its +behavior using the :paramref:`~.relationship.cascade_backrefs` flag, +is discussed in the section :ref:`backref_cascade`. + +.. _cascade_delete: + +delete +------ + +The ``delete`` cascade indicates that when a "parent" object +is marked for deletion, its related "child" objects should also be marked +for deletion. If for example we we have a relationship ``User.addresses`` +with ``delete`` cascade configured:: + + class User(Base): + # ... + + addresses = relationship("Address", cascade="save-update, merge, delete") + +If using the above mapping, we have a ``User`` object and two +related ``Address`` objects:: + + >>> user1 = sess.query(User).filter_by(id=1).first() + >>> address1, address2 = user1.addresses + +If we mark ``user1`` for deletion, after the flush operation proceeds, +``address1`` and ``address2`` will also be deleted: + +.. sourcecode:: python+sql + + >>> sess.delete(user1) + >>> sess.commit() + {opensql}DELETE FROM address WHERE address.id = ? + ((1,), (2,)) + DELETE FROM user WHERE user.id = ? + (1,) + COMMIT + +Alternatively, if our ``User.addresses`` relationship does *not* have +``delete`` cascade, SQLAlchemy's default behavior is to instead de-associate +``address1`` and ``address2`` from ``user1`` by setting their foreign key +reference to ``NULL``. Using a mapping as follows:: + + class User(Base): + # ... + + addresses = relationship("Address") + +Upon deletion of a parent ``User`` object, the rows in ``address`` are not +deleted, but are instead de-associated: + +.. sourcecode:: python+sql + + >>> sess.delete(user1) + >>> sess.commit() + {opensql}UPDATE address SET user_id=? WHERE address.id = ? + (None, 1) + UPDATE address SET user_id=? WHERE address.id = ? + (None, 2) + DELETE FROM user WHERE user.id = ? + (1,) + COMMIT + +``delete`` cascade is more often than not used in conjunction with +:ref:`cascade_delete_orphan` cascade, which will emit a DELETE for the related +row if the "child" object is deassociated from the parent. The combination +of ``delete`` and ``delete-orphan`` cascade covers both situations where +SQLAlchemy has to decide between setting a foreign key column to NULL versus +deleting the row entirely. + +.. topic:: ORM-level "delete" cascade vs. FOREIGN KEY level "ON DELETE" cascade + + The behavior of SQLAlchemy's "delete" cascade has a lot of overlap with the + ``ON DELETE CASCADE`` feature of a database foreign key, as well + as with that of the ``ON DELETE SET NULL`` foreign key setting when "delete" + cascade is not specified. Database level "ON DELETE" cascades are specific to the + "FOREIGN KEY" construct of the relational database; SQLAlchemy allows + configuration of these schema-level constructs at the :term:`DDL` level + using options on :class:`.ForeignKeyConstraint` which are described + at :ref:`on_update_on_delete`. + + It is important to note the differences between the ORM and the relational + database's notion of "cascade" as well as how they integrate: + + * A database level ``ON DELETE`` cascade is configured effectively + on the **many-to-one** side of the relationship; that is, we configure + it relative to the ``FOREIGN KEY`` constraint that is the "many" side + of a relationship. At the ORM level, **this direction is reversed**. + SQLAlchemy handles the deletion of "child" objects relative to a + "parent" from the "parent" side, which means that ``delete`` and + ``delete-orphan`` cascade are configured on the **one-to-many** + side. + + * Database level foreign keys with no ``ON DELETE`` setting + are often used to **prevent** a parent + row from being removed, as it would necessarily leave an unhandled + related row present. If this behavior is desired in a one-to-many + relationship, SQLAlchemy's default behavior of setting a foreign key + to ``NULL`` can be caught in one of two ways: + + * The easiest and most common is just to set the + foreign-key-holding column to ``NOT NULL`` at the database schema + level. An attempt by SQLAlchemy to set the column to NULL will + fail with a simple NOT NULL constraint exception. + + * The other, more special case way is to set the :paramref:`~.relationship.passive_deletes` + flag to the string ``"all"``. This has the effect of entirely + disabling SQLAlchemy's behavior of setting the foreign key column + to NULL, and a DELETE will be emitted for the parent row without + any affect on the child row, even if the child row is present + in memory. This may be desirable in the case when + database-level foreign key triggers, either special ``ON DELETE`` settings + or otherwise, need to be activated in all cases when a parent row is deleted. + + * Database level ``ON DELETE`` cascade is **vastly more efficient** + than that of SQLAlchemy. The database can chain a series of cascade + operations across many relationships at once; e.g. if row A is deleted, + all the related rows in table B can be deleted, and all the C rows related + to each of those B rows, and on and on, all within the scope of a single + DELETE statement. SQLAlchemy on the other hand, in order to support + the cascading delete operation fully, has to individually load each + related collection in order to target all rows that then may have further + related collections. That is, SQLAlchemy isn't sophisticated enough + to emit a DELETE for all those related rows at once within this context. + + * SQLAlchemy doesn't **need** to be this sophisticated, as we instead provide + smooth integration with the database's own ``ON DELETE`` functionality, + by using the :paramref:`~.relationship.passive_deletes` option in conjunction + with properly configured foreign key constraints. Under this behavior, + SQLAlchemy only emits DELETE for those rows that are already locally + present in the :class:`.Session`; for any collections that are unloaded, + it leaves them to the database to handle, rather than emitting a SELECT + for them. The section :ref:`passive_deletes` provides an example of this use. + + * While database-level ``ON DELETE`` functionality works only on the "many" + side of a relationship, SQLAlchemy's "delete" cascade + has **limited** ability to operate in the *reverse* direction as well, + meaning it can be configured on the "many" side to delete an object + on the "one" side when the reference on the "many" side is deleted. However + this can easily result in constraint violations if there are other objects + referring to this "one" side from the "many", so it typically is only + useful when a relationship is in fact a "one to one". The + :paramref:`~.relationship.single_parent` flag should be used to establish + an in-Python assertion for this case. + + +When using a :func:`.relationship` that also includes a many-to-many +table using the :paramref:`~.relationship.secondary` option, SQLAlchemy's +delete cascade handles the rows in this many-to-many table automatically. +Just like, as described in :ref:`relationships_many_to_many_deletion`, +the addition or removal of an object from a many-to-many collection +results in the INSERT or DELETE of a row in the many-to-many table, +the ``delete`` cascade, when activated as the result of a parent object +delete operation, will DELETE not just the row in the "child" table but also +in the many-to-many table. + +.. _cascade_delete_orphan: + +delete-orphan +------------- + +``delete-orphan`` cascade adds behavior to the ``delete`` cascade, +such that a child object will be marked for deletion when it is +de-associated from the parent, not just when the parent is marked +for deletion. This is a common feature when dealing with a related +object that is "owned" by its parent, with a NOT NULL foreign key, +so that removal of the item from the parent collection results +in its deletion. + +``delete-orphan`` cascade implies that each child object can only +have one parent at a time, so is configured in the vast majority of cases +on a one-to-many relationship. Setting it on a many-to-one or +many-to-many relationship is more awkward; for this use case, +SQLAlchemy requires that the :func:`~sqlalchemy.orm.relationship` +be configured with the :paramref:`~.relationship.single_parent` argument, +establishes Python-side validation that ensures the object +is associated with only one parent at a time. + +.. _cascade_merge: + +merge +----- + +``merge`` cascade indicates that the :meth:`.Session.merge` +operation should be propagated from a parent that's the subject +of the :meth:`.Session.merge` call down to referred objects. +This cascade is also on by default. + +.. _cascade_refresh_expire: + +refresh-expire +-------------- + +``refresh-expire`` is an uncommon option, indicating that the +:meth:`.Session.expire` operation should be propagated from a parent +down to referred objects. When using :meth:`.Session.refresh`, +the referred objects are expired only, but not actually refreshed. + +.. _cascade_expunge: + +expunge +------- + +``expunge`` cascade indicates that when the parent object is removed +from the :class:`.Session` using :meth:`.Session.expunge`, the +operation should be propagated down to referred objects. + +.. _backref_cascade: + +Controlling Cascade on Backrefs +------------------------------- + +The :ref:`cascade_save_update` cascade by default takes place on attribute change events +emitted from backrefs. This is probably a confusing statement more +easily described through demonstration; it means that, given a mapping such as this:: + + mapper(Order, order_table, properties={ + 'items' : relationship(Item, backref='order') + }) + +If an ``Order`` is already in the session, and is assigned to the ``order`` +attribute of an ``Item``, the backref appends the ``Order`` to the ``items`` +collection of that ``Order``, resulting in the ``save-update`` cascade taking +place:: + + >>> o1 = Order() + >>> session.add(o1) + >>> o1 in session + True + + >>> i1 = Item() + >>> i1.order = o1 + >>> i1 in o1.items + True + >>> i1 in session + True + +This behavior can be disabled using the :paramref:`~.relationship.cascade_backrefs` flag:: + + mapper(Order, order_table, properties={ + 'items' : relationship(Item, backref='order', + cascade_backrefs=False) + }) + +So above, the assignment of ``i1.order = o1`` will append ``i1`` to the ``items`` +collection of ``o1``, but will not add ``i1`` to the session. You can, of +course, :meth:`~.Session.add` ``i1`` to the session at a later point. This +option may be helpful for situations where an object needs to be kept out of a +session until it's construction is completed, but still needs to be given +associations to objects which are already persistent in the target session. diff --git a/doc/build/orm/classical.rst b/doc/build/orm/classical.rst new file mode 100644 index 000000000..0f04586c7 --- /dev/null +++ b/doc/build/orm/classical.rst @@ -0,0 +1,68 @@ +.. _classical_mapping: + +Classical Mappings +================== + +A *Classical Mapping* refers to the configuration of a mapped class using the +:func:`.mapper` function, without using the Declarative system. As an example, +start with the declarative mapping introduced in :ref:`ormtutorial_toplevel`:: + + class User(Base): + __tablename__ = 'users' + + id = Column(Integer, primary_key=True) + name = Column(String) + fullname = Column(String) + password = Column(String) + +In "classical" form, the table metadata is created separately with the :class:`.Table` +construct, then associated with the ``User`` class via the :func:`.mapper` function:: + + from sqlalchemy import Table, MetaData, Column, ForeignKey, Integer, String + from sqlalchemy.orm import mapper + + metadata = MetaData() + + user = Table('user', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)), + Column('fullname', String(50)), + Column('password', String(12)) + ) + + class User(object): + def __init__(self, name, fullname, password): + self.name = name + self.fullname = fullname + self.password = password + + mapper(User, user) + +Information about mapped attributes, such as relationships to other classes, are provided +via the ``properties`` dictionary. The example below illustrates a second :class:`.Table` +object, mapped to a class called ``Address``, then linked to ``User`` via :func:`.relationship`:: + + address = Table('address', metadata, + Column('id', Integer, primary_key=True), + Column('user_id', Integer, ForeignKey('user.id')), + Column('email_address', String(50)) + ) + + mapper(User, user, properties={ + 'addresses' : relationship(Address, backref='user', order_by=address.c.id) + }) + + mapper(Address, address) + +When using classical mappings, classes must be provided directly without the benefit +of the "string lookup" system provided by Declarative. SQL expressions are typically +specified in terms of the :class:`.Table` objects, i.e. ``address.c.id`` above +for the ``Address`` relationship, and not ``Address.id``, as ``Address`` may not +yet be linked to table metadata, nor can we specify a string here. + +Some examples in the documentation still use the classical approach, but note that +the classical as well as Declarative approaches are **fully interchangeable**. Both +systems ultimately create the same configuration, consisting of a :class:`.Table`, +user-defined class, linked together with a :func:`.mapper`. When we talk about +"the behavior of :func:`.mapper`", this includes when using the Declarative system +as well - it's still used, just behind the scenes. diff --git a/doc/build/orm/composites.rst b/doc/build/orm/composites.rst new file mode 100644 index 000000000..1c42564b1 --- /dev/null +++ b/doc/build/orm/composites.rst @@ -0,0 +1,160 @@ +.. module:: sqlalchemy.orm + +.. _mapper_composite: + +Composite Column Types +======================= + +Sets of columns can be associated with a single user-defined datatype. The ORM +provides a single attribute which represents the group of columns using the +class you provide. + +.. versionchanged:: 0.7 + Composites have been simplified such that + they no longer "conceal" the underlying column based attributes. Additionally, + in-place mutation is no longer automatic; see the section below on + enabling mutability to support tracking of in-place changes. + +.. versionchanged:: 0.9 + Composites will return their object-form, rather than as individual columns, + when used in a column-oriented :class:`.Query` construct. See :ref:`migration_2824`. + +A simple example represents pairs of columns as a ``Point`` object. +``Point`` represents such a pair as ``.x`` and ``.y``:: + + class Point(object): + def __init__(self, x, y): + self.x = x + self.y = y + + def __composite_values__(self): + return self.x, self.y + + def __repr__(self): + return "Point(x=%r, y=%r)" % (self.x, self.y) + + def __eq__(self, other): + return isinstance(other, Point) and \ + other.x == self.x and \ + other.y == self.y + + def __ne__(self, other): + return not self.__eq__(other) + +The requirements for the custom datatype class are that it have a constructor +which accepts positional arguments corresponding to its column format, and +also provides a method ``__composite_values__()`` which returns the state of +the object as a list or tuple, in order of its column-based attributes. It +also should supply adequate ``__eq__()`` and ``__ne__()`` methods which test +the equality of two instances. + +We will create a mapping to a table ``vertice``, which represents two points +as ``x1/y1`` and ``x2/y2``. These are created normally as :class:`.Column` +objects. Then, the :func:`.composite` function is used to assign new +attributes that will represent sets of columns via the ``Point`` class:: + + from sqlalchemy import Column, Integer + from sqlalchemy.orm import composite + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + class Vertex(Base): + __tablename__ = 'vertice' + + id = Column(Integer, primary_key=True) + x1 = Column(Integer) + y1 = Column(Integer) + x2 = Column(Integer) + y2 = Column(Integer) + + start = composite(Point, x1, y1) + end = composite(Point, x2, y2) + +A classical mapping above would define each :func:`.composite` +against the existing table:: + + mapper(Vertex, vertice_table, properties={ + 'start':composite(Point, vertice_table.c.x1, vertice_table.c.y1), + 'end':composite(Point, vertice_table.c.x2, vertice_table.c.y2), + }) + +We can now persist and use ``Vertex`` instances, as well as query for them, +using the ``.start`` and ``.end`` attributes against ad-hoc ``Point`` instances: + +.. sourcecode:: python+sql + + >>> v = Vertex(start=Point(3, 4), end=Point(5, 6)) + >>> session.add(v) + >>> q = session.query(Vertex).filter(Vertex.start == Point(3, 4)) + {sql}>>> print q.first().start + BEGIN (implicit) + INSERT INTO vertice (x1, y1, x2, y2) VALUES (?, ?, ?, ?) + (3, 4, 5, 6) + SELECT vertice.id AS vertice_id, + vertice.x1 AS vertice_x1, + vertice.y1 AS vertice_y1, + vertice.x2 AS vertice_x2, + vertice.y2 AS vertice_y2 + FROM vertice + WHERE vertice.x1 = ? AND vertice.y1 = ? + LIMIT ? OFFSET ? + (3, 4, 1, 0) + {stop}Point(x=3, y=4) + +.. autofunction:: composite + + +Tracking In-Place Mutations on Composites +----------------------------------------- + +In-place changes to an existing composite value are +not tracked automatically. Instead, the composite class needs to provide +events to its parent object explicitly. This task is largely automated +via the usage of the :class:`.MutableComposite` mixin, which uses events +to associate each user-defined composite object with all parent associations. +Please see the example in :ref:`mutable_composites`. + +.. versionchanged:: 0.7 + In-place changes to an existing composite value are no longer + tracked automatically; the functionality is superseded by the + :class:`.MutableComposite` class. + +.. _composite_operations: + +Redefining Comparison Operations for Composites +----------------------------------------------- + +The "equals" comparison operation by default produces an AND of all +corresponding columns equated to one another. This can be changed using +the ``comparator_factory`` argument to :func:`.composite`, where we +specify a custom :class:`.CompositeProperty.Comparator` class +to define existing or new operations. +Below we illustrate the "greater than" operator, implementing +the same expression that the base "greater than" does:: + + from sqlalchemy.orm.properties import CompositeProperty + from sqlalchemy import sql + + class PointComparator(CompositeProperty.Comparator): + def __gt__(self, other): + """redefine the 'greater than' operation""" + + return sql.and_(*[a>b for a, b in + zip(self.__clause_element__().clauses, + other.__composite_values__())]) + + class Vertex(Base): + ___tablename__ = 'vertice' + + id = Column(Integer, primary_key=True) + x1 = Column(Integer) + y1 = Column(Integer) + x2 = Column(Integer) + y2 = Column(Integer) + + start = composite(Point, x1, y1, + comparator_factory=PointComparator) + end = composite(Point, x2, y2, + comparator_factory=PointComparator) + diff --git a/doc/build/orm/constructors.rst b/doc/build/orm/constructors.rst new file mode 100644 index 000000000..ab6691553 --- /dev/null +++ b/doc/build/orm/constructors.rst @@ -0,0 +1,56 @@ +.. _mapping_constructors: + +Constructors and Object Initialization +======================================= + +Mapping imposes no restrictions or requirements on the constructor +(``__init__``) method for the class. You are free to require any arguments for +the function that you wish, assign attributes to the instance that are unknown +to the ORM, and generally do anything else you would normally do when writing +a constructor for a Python class. + +The SQLAlchemy ORM does not call ``__init__`` when recreating objects from +database rows. The ORM's process is somewhat akin to the Python standard +library's ``pickle`` module, invoking the low level ``__new__`` method and +then quietly restoring attributes directly on the instance rather than calling +``__init__``. + +If you need to do some setup on database-loaded instances before they're ready +to use, you can use the ``@reconstructor`` decorator to tag a method as the +ORM counterpart to ``__init__``. SQLAlchemy will call this method with no +arguments every time it loads or reconstructs one of your instances. This is +useful for recreating transient properties that are normally assigned in your +``__init__``:: + + from sqlalchemy import orm + + class MyMappedClass(object): + def __init__(self, data): + self.data = data + # we need stuff on all instances, but not in the database. + self.stuff = [] + + @orm.reconstructor + def init_on_load(self): + self.stuff = [] + +When ``obj = MyMappedClass()`` is executed, Python calls the ``__init__`` +method as normal and the ``data`` argument is required. When instances are +loaded during a :class:`~sqlalchemy.orm.query.Query` operation as in +``query(MyMappedClass).one()``, ``init_on_load`` is called. + +Any method may be tagged as the :func:`~sqlalchemy.orm.reconstructor`, even +the ``__init__`` method. SQLAlchemy will call the reconstructor method with no +arguments. Scalar (non-collection) database-mapped attributes of the instance +will be available for use within the function. Eagerly-loaded collections are +generally not yet available and will usually only contain the first element. +ORM state changes made to objects at this stage will not be recorded for the +next flush() operation, so the activity within a reconstructor should be +conservative. + +:func:`~sqlalchemy.orm.reconstructor` is a shortcut into a larger system +of "instance level" events, which can be subscribed to using the +event API - see :class:`.InstanceEvents` for the full API description +of these events. + +.. autofunction:: reconstructor diff --git a/doc/build/orm/contextual.rst b/doc/build/orm/contextual.rst new file mode 100644 index 000000000..cc7016f80 --- /dev/null +++ b/doc/build/orm/contextual.rst @@ -0,0 +1,260 @@ +.. _unitofwork_contextual: + +Contextual/Thread-local Sessions +================================= + +Recall from the section :ref:`session_faq_whentocreate`, the concept of +"session scopes" was introduced, with an emphasis on web applications +and the practice of linking the scope of a :class:`.Session` with that +of a web request. Most modern web frameworks include integration tools +so that the scope of the :class:`.Session` can be managed automatically, +and these tools should be used as they are available. + +SQLAlchemy includes its own helper object, which helps with the establishment +of user-defined :class:`.Session` scopes. It is also used by third-party +integration systems to help construct their integration schemes. + +The object is the :class:`.scoped_session` object, and it represents a +**registry** of :class:`.Session` objects. If you're not familiar with the +registry pattern, a good introduction can be found in `Patterns of Enterprise +Architecture `_. + +.. note:: + + The :class:`.scoped_session` object is a very popular and useful object + used by many SQLAlchemy applications. However, it is important to note + that it presents **only one approach** to the issue of :class:`.Session` + management. If you're new to SQLAlchemy, and especially if the + term "thread-local variable" seems strange to you, we recommend that + if possible you familiarize first with an off-the-shelf integration + system such as `Flask-SQLAlchemy `_ + or `zope.sqlalchemy `_. + +A :class:`.scoped_session` is constructed by calling it, passing it a +**factory** which can create new :class:`.Session` objects. A factory +is just something that produces a new object when called, and in the +case of :class:`.Session`, the most common factory is the :class:`.sessionmaker`, +introduced earlier in this section. Below we illustrate this usage:: + + >>> from sqlalchemy.orm import scoped_session + >>> from sqlalchemy.orm import sessionmaker + + >>> session_factory = sessionmaker(bind=some_engine) + >>> Session = scoped_session(session_factory) + +The :class:`.scoped_session` object we've created will now call upon the +:class:`.sessionmaker` when we "call" the registry:: + + >>> some_session = Session() + +Above, ``some_session`` is an instance of :class:`.Session`, which we +can now use to talk to the database. This same :class:`.Session` is also +present within the :class:`.scoped_session` registry we've created. If +we call upon the registry a second time, we get back the **same** :class:`.Session`:: + + >>> some_other_session = Session() + >>> some_session is some_other_session + True + +This pattern allows disparate sections of the application to call upon a global +:class:`.scoped_session`, so that all those areas may share the same session +without the need to pass it explicitly. The :class:`.Session` we've established +in our registry will remain, until we explicitly tell our registry to dispose of it, +by calling :meth:`.scoped_session.remove`:: + + >>> Session.remove() + +The :meth:`.scoped_session.remove` method first calls :meth:`.Session.close` on +the current :class:`.Session`, which has the effect of releasing any connection/transactional +resources owned by the :class:`.Session` first, then discarding the :class:`.Session` +itself. "Releasing" here means that connections are returned to their connection pool and any transactional state is rolled back, ultimately using the ``rollback()`` method of the underlying DBAPI connection. + +At this point, the :class:`.scoped_session` object is "empty", and will create +a **new** :class:`.Session` when called again. As illustrated below, this +is not the same :class:`.Session` we had before:: + + >>> new_session = Session() + >>> new_session is some_session + False + +The above series of steps illustrates the idea of the "registry" pattern in a +nutshell. With that basic idea in hand, we can discuss some of the details +of how this pattern proceeds. + +Implicit Method Access +---------------------- + +The job of the :class:`.scoped_session` is simple; hold onto a :class:`.Session` +for all who ask for it. As a means of producing more transparent access to this +:class:`.Session`, the :class:`.scoped_session` also includes **proxy behavior**, +meaning that the registry itself can be treated just like a :class:`.Session` +directly; when methods are called on this object, they are **proxied** to the +underlying :class:`.Session` being maintained by the registry:: + + Session = scoped_session(some_factory) + + # equivalent to: + # + # session = Session() + # print session.query(MyClass).all() + # + print Session.query(MyClass).all() + +The above code accomplishes the same task as that of acquiring the current +:class:`.Session` by calling upon the registry, then using that :class:`.Session`. + +Thread-Local Scope +------------------ + +Users who are familiar with multithreaded programming will note that representing +anything as a global variable is usually a bad idea, as it implies that the +global object will be accessed by many threads concurrently. The :class:`.Session` +object is entirely designed to be used in a **non-concurrent** fashion, which +in terms of multithreading means "only in one thread at a time". So our +above example of :class:`.scoped_session` usage, where the same :class:`.Session` +object is maintained across multiple calls, suggests that some process needs +to be in place such that mutltiple calls across many threads don't actually get +a handle to the same session. We call this notion **thread local storage**, +which means, a special object is used that will maintain a distinct object +per each application thread. Python provides this via the +`threading.local() `_ +construct. The :class:`.scoped_session` object by default uses this object +as storage, so that a single :class:`.Session` is maintained for all who call +upon the :class:`.scoped_session` registry, but only within the scope of a single +thread. Callers who call upon the registry in a different thread get a +:class:`.Session` instance that is local to that other thread. + +Using this technique, the :class:`.scoped_session` provides a quick and relatively +simple (if one is familiar with thread-local storage) way of providing +a single, global object in an application that is safe to be called upon +from multiple threads. + +The :meth:`.scoped_session.remove` method, as always, removes the current +:class:`.Session` associated with the thread, if any. However, one advantage of the +``threading.local()`` object is that if the application thread itself ends, the +"storage" for that thread is also garbage collected. So it is in fact "safe" to +use thread local scope with an application that spawns and tears down threads, +without the need to call :meth:`.scoped_session.remove`. However, the scope +of transactions themselves, i.e. ending them via :meth:`.Session.commit` or +:meth:`.Session.rollback`, will usually still be something that must be explicitly +arranged for at the appropriate time, unless the application actually ties the +lifespan of a thread to the lifespan of a transaction. + +.. _session_lifespan: + +Using Thread-Local Scope with Web Applications +---------------------------------------------- + +As discussed in the section :ref:`session_faq_whentocreate`, a web application +is architected around the concept of a **web request**, and integrating +such an application with the :class:`.Session` usually implies that the :class:`.Session` +will be associated with that request. As it turns out, most Python web frameworks, +with notable exceptions such as the asynchronous frameworks Twisted and +Tornado, use threads in a simple way, such that a particular web request is received, +processed, and completed within the scope of a single *worker thread*. When +the request ends, the worker thread is released to a pool of workers where it +is available to handle another request. + +This simple correspondence of web request and thread means that to associate a +:class:`.Session` with a thread implies it is also associated with the web request +running within that thread, and vice versa, provided that the :class:`.Session` is +created only after the web request begins and torn down just before the web request ends. +So it is a common practice to use :class:`.scoped_session` as a quick way +to integrate the :class:`.Session` with a web application. The sequence +diagram below illustrates this flow:: + + Web Server Web Framework SQLAlchemy ORM Code + -------------- -------------- ------------------------------ + startup -> Web framework # Session registry is established + initializes Session = scoped_session(sessionmaker()) + + incoming + web request -> web request -> # The registry is *optionally* + starts # called upon explicitly to create + # a Session local to the thread and/or request + Session() + + # the Session registry can otherwise + # be used at any time, creating the + # request-local Session() if not present, + # or returning the existing one + Session.query(MyClass) # ... + + Session.add(some_object) # ... + + # if data was modified, commit the + # transaction + Session.commit() + + web request ends -> # the registry is instructed to + # remove the Session + Session.remove() + + sends output <- + outgoing web <- + response + +Using the above flow, the process of integrating the :class:`.Session` with the +web application has exactly two requirements: + +1. Create a single :class:`.scoped_session` registry when the web application + first starts, ensuring that this object is accessible by the rest of the + application. +2. Ensure that :meth:`.scoped_session.remove` is called when the web request ends, + usually by integrating with the web framework's event system to establish + an "on request end" event. + +As noted earlier, the above pattern is **just one potential way** to integrate a :class:`.Session` +with a web framework, one which in particular makes the significant assumption +that the **web framework associates web requests with application threads**. It is +however **strongly recommended that the integration tools provided with the web framework +itself be used, if available**, instead of :class:`.scoped_session`. + +In particular, while using a thread local can be convenient, it is preferable that the :class:`.Session` be +associated **directly with the request**, rather than with +the current thread. The next section on custom scopes details a more advanced configuration +which can combine the usage of :class:`.scoped_session` with direct request based scope, or +any kind of scope. + +Using Custom Created Scopes +--------------------------- + +The :class:`.scoped_session` object's default behavior of "thread local" scope is only +one of many options on how to "scope" a :class:`.Session`. A custom scope can be defined +based on any existing system of getting at "the current thing we are working with". + +Suppose a web framework defines a library function ``get_current_request()``. An application +built using this framework can call this function at any time, and the result will be +some kind of ``Request`` object that represents the current request being processed. +If the ``Request`` object is hashable, then this function can be easily integrated with +:class:`.scoped_session` to associate the :class:`.Session` with the request. Below we illustrate +this in conjunction with a hypothetical event marker provided by the web framework +``on_request_end``, which allows code to be invoked whenever a request ends:: + + from my_web_framework import get_current_request, on_request_end + from sqlalchemy.orm import scoped_session, sessionmaker + + Session = scoped_session(sessionmaker(bind=some_engine), scopefunc=get_current_request) + + @on_request_end + def remove_session(req): + Session.remove() + +Above, we instantiate :class:`.scoped_session` in the usual way, except that we pass +our request-returning function as the "scopefunc". This instructs :class:`.scoped_session` +to use this function to generate a dictionary key whenever the registry is called upon +to return the current :class:`.Session`. In this case it is particularly important +that we ensure a reliable "remove" system is implemented, as this dictionary is not +otherwise self-managed. + + +Contextual Session API +---------------------- + +.. autoclass:: sqlalchemy.orm.scoping.scoped_session + :members: + +.. autoclass:: sqlalchemy.util.ScopedRegistry + :members: + +.. autoclass:: sqlalchemy.util.ThreadLocalRegistry diff --git a/doc/build/orm/extending.rst b/doc/build/orm/extending.rst new file mode 100644 index 000000000..4b2b86f62 --- /dev/null +++ b/doc/build/orm/extending.rst @@ -0,0 +1,12 @@ +==================== +Events and Internals +==================== + +.. toctree:: + :maxdepth: 2 + + events + internals + exceptions + deprecated + diff --git a/doc/build/orm/extensions/declarative.rst b/doc/build/orm/extensions/declarative.rst deleted file mode 100644 index 7d9e634b5..000000000 --- a/doc/build/orm/extensions/declarative.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. _declarative_toplevel: - -Declarative -=========== - -.. automodule:: sqlalchemy.ext.declarative - -API Reference -------------- - -.. autofunction:: declarative_base - -.. autofunction:: as_declarative - -.. autoclass:: declared_attr - :members: - -.. autofunction:: sqlalchemy.ext.declarative.api._declarative_constructor - -.. autofunction:: has_inherited_table - -.. autofunction:: synonym_for - -.. autofunction:: comparable_using - -.. autofunction:: instrument_declarative - -.. autoclass:: AbstractConcreteBase - -.. autoclass:: ConcreteBase - -.. autoclass:: DeferredReflection - :members: diff --git a/doc/build/orm/extensions/declarative/api.rst b/doc/build/orm/extensions/declarative/api.rst new file mode 100644 index 000000000..67b66a970 --- /dev/null +++ b/doc/build/orm/extensions/declarative/api.rst @@ -0,0 +1,114 @@ +.. automodule:: sqlalchemy.ext.declarative + +=============== +Declarative API +=============== + +API Reference +============= + +.. autofunction:: declarative_base + +.. autofunction:: as_declarative + +.. autoclass:: declared_attr + :members: + +.. autofunction:: sqlalchemy.ext.declarative.api._declarative_constructor + +.. autofunction:: has_inherited_table + +.. autofunction:: synonym_for + +.. autofunction:: comparable_using + +.. autofunction:: instrument_declarative + +.. autoclass:: AbstractConcreteBase + +.. autoclass:: ConcreteBase + +.. autoclass:: DeferredReflection + :members: + + +Special Directives +------------------ + +``__declare_last__()`` +~~~~~~~~~~~~~~~~~~~~~~ + +The ``__declare_last__()`` hook allows definition of +a class level function that is automatically called by the +:meth:`.MapperEvents.after_configured` event, which occurs after mappings are +assumed to be completed and the 'configure' step has finished:: + + class MyClass(Base): + @classmethod + def __declare_last__(cls): + "" + # do something with mappings + +.. versionadded:: 0.7.3 + +``__declare_first__()`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Like ``__declare_last__()``, but is called at the beginning of mapper +configuration via the :meth:`.MapperEvents.before_configured` event:: + + class MyClass(Base): + @classmethod + def __declare_first__(cls): + "" + # do something before mappings are configured + +.. versionadded:: 0.9.3 + +.. _declarative_abstract: + +``__abstract__`` +~~~~~~~~~~~~~~~~~~~ + +``__abstract__`` causes declarative to skip the production +of a table or mapper for the class entirely. A class can be added within a +hierarchy in the same way as mixin (see :ref:`declarative_mixins`), allowing +subclasses to extend just from the special class:: + + class SomeAbstractBase(Base): + __abstract__ = True + + def some_helpful_method(self): + "" + + @declared_attr + def __mapper_args__(cls): + return {"helpful mapper arguments":True} + + class MyMappedClass(SomeAbstractBase): + "" + +One possible use of ``__abstract__`` is to use a distinct +:class:`.MetaData` for different bases:: + + Base = declarative_base() + + class DefaultBase(Base): + __abstract__ = True + metadata = MetaData() + + class OtherBase(Base): + __abstract__ = True + metadata = MetaData() + +Above, classes which inherit from ``DefaultBase`` will use one +:class:`.MetaData` as the registry of tables, and those which inherit from +``OtherBase`` will use a different one. The tables themselves can then be +created perhaps within distinct databases:: + + DefaultBase.metadata.create_all(some_engine) + OtherBase.metadata_create_all(some_other_engine) + +.. versionadded:: 0.7.3 + + diff --git a/doc/build/orm/extensions/declarative/basic_use.rst b/doc/build/orm/extensions/declarative/basic_use.rst new file mode 100644 index 000000000..10b79e5a6 --- /dev/null +++ b/doc/build/orm/extensions/declarative/basic_use.rst @@ -0,0 +1,133 @@ +========= +Basic Use +========= + +SQLAlchemy object-relational configuration involves the +combination of :class:`.Table`, :func:`.mapper`, and class +objects to define a mapped class. +:mod:`~sqlalchemy.ext.declarative` allows all three to be +expressed at once within the class declaration. As much as +possible, regular SQLAlchemy schema and ORM constructs are +used directly, so that configuration between "classical" ORM +usage and declarative remain highly similar. + +As a simple example:: + + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + class SomeClass(Base): + __tablename__ = 'some_table' + id = Column(Integer, primary_key=True) + name = Column(String(50)) + +Above, the :func:`declarative_base` callable returns a new base class from +which all mapped classes should inherit. When the class definition is +completed, a new :class:`.Table` and :func:`.mapper` will have been generated. + +The resulting table and mapper are accessible via +``__table__`` and ``__mapper__`` attributes on the +``SomeClass`` class:: + + # access the mapped Table + SomeClass.__table__ + + # access the Mapper + SomeClass.__mapper__ + +Defining Attributes +=================== + +In the previous example, the :class:`.Column` objects are +automatically named with the name of the attribute to which they are +assigned. + +To name columns explicitly with a name distinct from their mapped attribute, +just give the column a name. Below, column "some_table_id" is mapped to the +"id" attribute of `SomeClass`, but in SQL will be represented as +"some_table_id":: + + class SomeClass(Base): + __tablename__ = 'some_table' + id = Column("some_table_id", Integer, primary_key=True) + +Attributes may be added to the class after its construction, and they will be +added to the underlying :class:`.Table` and +:func:`.mapper` definitions as appropriate:: + + SomeClass.data = Column('data', Unicode) + SomeClass.related = relationship(RelatedInfo) + +Classes which are constructed using declarative can interact freely +with classes that are mapped explicitly with :func:`.mapper`. + +It is recommended, though not required, that all tables +share the same underlying :class:`~sqlalchemy.schema.MetaData` object, +so that string-configured :class:`~sqlalchemy.schema.ForeignKey` +references can be resolved without issue. + +Accessing the MetaData +======================= + +The :func:`declarative_base` base class contains a +:class:`.MetaData` object where newly defined +:class:`.Table` objects are collected. This object is +intended to be accessed directly for +:class:`.MetaData`-specific operations. Such as, to issue +CREATE statements for all tables:: + + engine = create_engine('sqlite://') + Base.metadata.create_all(engine) + +:func:`declarative_base` can also receive a pre-existing +:class:`.MetaData` object, which allows a +declarative setup to be associated with an already +existing traditional collection of :class:`~sqlalchemy.schema.Table` +objects:: + + mymetadata = MetaData() + Base = declarative_base(metadata=mymetadata) + + +Class Constructor +================= + +As a convenience feature, the :func:`declarative_base` sets a default +constructor on classes which takes keyword arguments, and assigns them +to the named attributes:: + + e = Engineer(primary_language='python') + +Mapper Configuration +==================== + +Declarative makes use of the :func:`~.orm.mapper` function internally +when it creates the mapping to the declared table. The options +for :func:`~.orm.mapper` are passed directly through via the +``__mapper_args__`` class attribute. As always, arguments which reference +locally mapped columns can reference them directly from within the +class declaration:: + + from datetime import datetime + + class Widget(Base): + __tablename__ = 'widgets' + + id = Column(Integer, primary_key=True) + timestamp = Column(DateTime, nullable=False) + + __mapper_args__ = { + 'version_id_col': timestamp, + 'version_id_generator': lambda v:datetime.now() + } + + +.. _declarative_sql_expressions: + +Defining SQL Expressions +======================== + +See :ref:`mapper_sql_expressions` for examples on declaratively +mapping attributes to SQL expressions. + diff --git a/doc/build/orm/extensions/declarative/index.rst b/doc/build/orm/extensions/declarative/index.rst new file mode 100644 index 000000000..dc4f392f3 --- /dev/null +++ b/doc/build/orm/extensions/declarative/index.rst @@ -0,0 +1,32 @@ +.. _declarative_toplevel: + +=========== +Declarative +=========== + +The Declarative system is the typically used system provided by the SQLAlchemy +ORM in order to define classes mapped to relational database tables. However, +as noted in :ref:`classical_mapping`, Declarative is in fact a series of +extensions that ride on top of the SQLAlchemy :func:`.mapper` construct. + +While the documentation typically refers to Declarative for most examples, +the following sections will provide detailed information on how the +Declarative API interacts with the basic :func:`.mapper` and Core :class:`.Table` +systems, as well as how sophisticated patterns can be built using systems +such as mixins. + + +.. toctree:: + :maxdepth: 2 + + basic_use + relationships + table_config + inheritance + mixins + api + + + + + diff --git a/doc/build/orm/extensions/declarative/inheritance.rst b/doc/build/orm/extensions/declarative/inheritance.rst new file mode 100644 index 000000000..684b07bfd --- /dev/null +++ b/doc/build/orm/extensions/declarative/inheritance.rst @@ -0,0 +1,318 @@ +.. _declarative_inheritance: + +Inheritance Configuration +========================= + +Declarative supports all three forms of inheritance as intuitively +as possible. The ``inherits`` mapper keyword argument is not needed +as declarative will determine this from the class itself. The various +"polymorphic" keyword arguments are specified using ``__mapper_args__``. + +Joined Table Inheritance +~~~~~~~~~~~~~~~~~~~~~~~~ + +Joined table inheritance is defined as a subclass that defines its own +table:: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __tablename__ = 'engineers' + __mapper_args__ = {'polymorphic_identity': 'engineer'} + id = Column(Integer, ForeignKey('people.id'), primary_key=True) + primary_language = Column(String(50)) + +Note that above, the ``Engineer.id`` attribute, since it shares the +same attribute name as the ``Person.id`` attribute, will in fact +represent the ``people.id`` and ``engineers.id`` columns together, +with the "Engineer.id" column taking precedence if queried directly. +To provide the ``Engineer`` class with an attribute that represents +only the ``engineers.id`` column, give it a different attribute name:: + + class Engineer(Person): + __tablename__ = 'engineers' + __mapper_args__ = {'polymorphic_identity': 'engineer'} + engineer_id = Column('id', Integer, ForeignKey('people.id'), + primary_key=True) + primary_language = Column(String(50)) + + +.. versionchanged:: 0.7 joined table inheritance favors the subclass + column over that of the superclass, such as querying above + for ``Engineer.id``. Prior to 0.7 this was the reverse. + +.. _declarative_single_table: + +Single Table Inheritance +~~~~~~~~~~~~~~~~~~~~~~~~ + +Single table inheritance is defined as a subclass that does not have +its own table; you just leave out the ``__table__`` and ``__tablename__`` +attributes:: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + primary_language = Column(String(50)) + +When the above mappers are configured, the ``Person`` class is mapped +to the ``people`` table *before* the ``primary_language`` column is +defined, and this column will not be included in its own mapping. +When ``Engineer`` then defines the ``primary_language`` column, the +column is added to the ``people`` table so that it is included in the +mapping for ``Engineer`` and is also part of the table's full set of +columns. Columns which are not mapped to ``Person`` are also excluded +from any other single or joined inheriting classes using the +``exclude_properties`` mapper argument. Below, ``Manager`` will have +all the attributes of ``Person`` and ``Manager`` but *not* the +``primary_language`` attribute of ``Engineer``:: + + class Manager(Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + golf_swing = Column(String(50)) + +The attribute exclusion logic is provided by the +``exclude_properties`` mapper argument, and declarative's default +behavior can be disabled by passing an explicit ``exclude_properties`` +collection (empty or otherwise) to the ``__mapper_args__``. + +Resolving Column Conflicts +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note above that the ``primary_language`` and ``golf_swing`` columns +are "moved up" to be applied to ``Person.__table__``, as a result of their +declaration on a subclass that has no table of its own. A tricky case +comes up when two subclasses want to specify *the same* column, as below:: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + start_date = Column(DateTime) + + class Manager(Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + start_date = Column(DateTime) + +Above, the ``start_date`` column declared on both ``Engineer`` and ``Manager`` +will result in an error:: + + sqlalchemy.exc.ArgumentError: Column 'start_date' on class + conflicts with existing + column 'people.start_date' + +In a situation like this, Declarative can't be sure +of the intent, especially if the ``start_date`` columns had, for example, +different types. A situation like this can be resolved by using +:class:`.declared_attr` to define the :class:`.Column` conditionally, taking +care to return the **existing column** via the parent ``__table__`` if it +already exists:: + + from sqlalchemy.ext.declarative import declared_attr + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + + @declared_attr + def start_date(cls): + "Start date column, if not present already." + return Person.__table__.c.get('start_date', Column(DateTime)) + + class Manager(Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + + @declared_attr + def start_date(cls): + "Start date column, if not present already." + return Person.__table__.c.get('start_date', Column(DateTime)) + +Above, when ``Manager`` is mapped, the ``start_date`` column is +already present on the ``Person`` class. Declarative lets us return +that :class:`.Column` as a result in this case, where it knows to skip +re-assigning the same column. If the mapping is mis-configured such +that the ``start_date`` column is accidentally re-assigned to a +different table (such as, if we changed ``Manager`` to be joined +inheritance without fixing ``start_date``), an error is raised which +indicates an existing :class:`.Column` is trying to be re-assigned to +a different owning :class:`.Table`. + +.. versionadded:: 0.8 :class:`.declared_attr` can be used on a non-mixin + class, and the returned :class:`.Column` or other mapped attribute + will be applied to the mapping as any other attribute. Previously, + the resulting attribute would be ignored, and also result in a warning + being emitted when a subclass was created. + +.. versionadded:: 0.8 :class:`.declared_attr`, when used either with a + mixin or non-mixin declarative class, can return an existing + :class:`.Column` already assigned to the parent :class:`.Table`, + to indicate that the re-assignment of the :class:`.Column` should be + skipped, however should still be mapped on the target class, + in order to resolve duplicate column conflicts. + +The same concept can be used with mixin classes (see +:ref:`declarative_mixins`):: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class HasStartDate(object): + @declared_attr + def start_date(cls): + return cls.__table__.c.get('start_date', Column(DateTime)) + + class Engineer(HasStartDate, Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + + class Manager(HasStartDate, Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + +The above mixin checks the local ``__table__`` attribute for the column. +Because we're using single table inheritance, we're sure that in this case, +``cls.__table__`` refers to ``People.__table__``. If we were mixing joined- +and single-table inheritance, we might want our mixin to check more carefully +if ``cls.__table__`` is really the :class:`.Table` we're looking for. + +Concrete Table Inheritance +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Concrete is defined as a subclass which has its own table and sets the +``concrete`` keyword argument to ``True``:: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + name = Column(String(50)) + + class Engineer(Person): + __tablename__ = 'engineers' + __mapper_args__ = {'concrete':True} + id = Column(Integer, primary_key=True) + primary_language = Column(String(50)) + name = Column(String(50)) + +Usage of an abstract base class is a little less straightforward as it +requires usage of :func:`~sqlalchemy.orm.util.polymorphic_union`, +which needs to be created with the :class:`.Table` objects +before the class is built:: + + engineers = Table('engineers', Base.metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)), + Column('primary_language', String(50)) + ) + managers = Table('managers', Base.metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)), + Column('golf_swing', String(50)) + ) + + punion = polymorphic_union({ + 'engineer':engineers, + 'manager':managers + }, 'type', 'punion') + + class Person(Base): + __table__ = punion + __mapper_args__ = {'polymorphic_on':punion.c.type} + + class Engineer(Person): + __table__ = engineers + __mapper_args__ = {'polymorphic_identity':'engineer', 'concrete':True} + + class Manager(Person): + __table__ = managers + __mapper_args__ = {'polymorphic_identity':'manager', 'concrete':True} + +.. _declarative_concrete_helpers: + +Using the Concrete Helpers +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Helper classes provides a simpler pattern for concrete inheritance. +With these objects, the ``__declare_first__`` helper is used to configure the +"polymorphic" loader for the mapper after all subclasses have been declared. + +.. versionadded:: 0.7.3 + +An abstract base can be declared using the +:class:`.AbstractConcreteBase` class:: + + from sqlalchemy.ext.declarative import AbstractConcreteBase + + class Employee(AbstractConcreteBase, Base): + pass + +To have a concrete ``employee`` table, use :class:`.ConcreteBase` instead:: + + from sqlalchemy.ext.declarative import ConcreteBase + + class Employee(ConcreteBase, Base): + __tablename__ = 'employee' + employee_id = Column(Integer, primary_key=True) + name = Column(String(50)) + __mapper_args__ = { + 'polymorphic_identity':'employee', + 'concrete':True} + + +Either ``Employee`` base can be used in the normal fashion:: + + class Manager(Employee): + __tablename__ = 'manager' + employee_id = Column(Integer, primary_key=True) + name = Column(String(50)) + manager_data = Column(String(40)) + __mapper_args__ = { + 'polymorphic_identity':'manager', + 'concrete':True} + + class Engineer(Employee): + __tablename__ = 'engineer' + employee_id = Column(Integer, primary_key=True) + name = Column(String(50)) + engineer_info = Column(String(40)) + __mapper_args__ = {'polymorphic_identity':'engineer', + 'concrete':True} + + +The :class:`.AbstractConcreteBase` class is itself mapped, and can be +used as a target of relationships:: + + class Company(Base): + __tablename__ = 'company' + + id = Column(Integer, primary_key=True) + employees = relationship("Employee", + primaryjoin="Company.id == Employee.company_id") + + +.. versionchanged:: 0.9.3 Support for use of :class:`.AbstractConcreteBase` + as the target of a :func:`.relationship` has been improved. + +It can also be queried directly:: + + for employee in session.query(Employee).filter(Employee.name == 'qbert'): + print(employee) + diff --git a/doc/build/orm/extensions/declarative/mixins.rst b/doc/build/orm/extensions/declarative/mixins.rst new file mode 100644 index 000000000..d64477649 --- /dev/null +++ b/doc/build/orm/extensions/declarative/mixins.rst @@ -0,0 +1,541 @@ +.. _declarative_mixins: + +Mixin and Custom Base Classes +============================== + +A common need when using :mod:`~sqlalchemy.ext.declarative` is to +share some functionality, such as a set of common columns, some common +table options, or other mapped properties, across many +classes. The standard Python idioms for this is to have the classes +inherit from a base which includes these common features. + +When using :mod:`~sqlalchemy.ext.declarative`, this idiom is allowed +via the usage of a custom declarative base class, as well as a "mixin" class +which is inherited from in addition to the primary base. Declarative +includes several helper features to make this work in terms of how +mappings are declared. An example of some commonly mixed-in +idioms is below:: + + from sqlalchemy.ext.declarative import declared_attr + + class MyMixin(object): + + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() + + __table_args__ = {'mysql_engine': 'InnoDB'} + __mapper_args__= {'always_refresh': True} + + id = Column(Integer, primary_key=True) + + class MyModel(MyMixin, Base): + name = Column(String(1000)) + +Where above, the class ``MyModel`` will contain an "id" column +as the primary key, a ``__tablename__`` attribute that derives +from the name of the class itself, as well as ``__table_args__`` +and ``__mapper_args__`` defined by the ``MyMixin`` mixin class. + +There's no fixed convention over whether ``MyMixin`` precedes +``Base`` or not. Normal Python method resolution rules apply, and +the above example would work just as well with:: + + class MyModel(Base, MyMixin): + name = Column(String(1000)) + +This works because ``Base`` here doesn't define any of the +variables that ``MyMixin`` defines, i.e. ``__tablename__``, +``__table_args__``, ``id``, etc. If the ``Base`` did define +an attribute of the same name, the class placed first in the +inherits list would determine which attribute is used on the +newly defined class. + +Augmenting the Base +~~~~~~~~~~~~~~~~~~~ + +In addition to using a pure mixin, most of the techniques in this +section can also be applied to the base class itself, for patterns that +should apply to all classes derived from a particular base. This is achieved +using the ``cls`` argument of the :func:`.declarative_base` function:: + + from sqlalchemy.ext.declarative import declared_attr + + class Base(object): + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() + + __table_args__ = {'mysql_engine': 'InnoDB'} + + id = Column(Integer, primary_key=True) + + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base(cls=Base) + + class MyModel(Base): + name = Column(String(1000)) + +Where above, ``MyModel`` and all other classes that derive from ``Base`` will +have a table name derived from the class name, an ``id`` primary key column, +as well as the "InnoDB" engine for MySQL. + +Mixing in Columns +~~~~~~~~~~~~~~~~~ + +The most basic way to specify a column on a mixin is by simple +declaration:: + + class TimestampMixin(object): + created_at = Column(DateTime, default=func.now()) + + class MyModel(TimestampMixin, Base): + __tablename__ = 'test' + + id = Column(Integer, primary_key=True) + name = Column(String(1000)) + +Where above, all declarative classes that include ``TimestampMixin`` +will also have a column ``created_at`` that applies a timestamp to +all row insertions. + +Those familiar with the SQLAlchemy expression language know that +the object identity of clause elements defines their role in a schema. +Two ``Table`` objects ``a`` and ``b`` may both have a column called +``id``, but the way these are differentiated is that ``a.c.id`` +and ``b.c.id`` are two distinct Python objects, referencing their +parent tables ``a`` and ``b`` respectively. + +In the case of the mixin column, it seems that only one +:class:`.Column` object is explicitly created, yet the ultimate +``created_at`` column above must exist as a distinct Python object +for each separate destination class. To accomplish this, the declarative +extension creates a **copy** of each :class:`.Column` object encountered on +a class that is detected as a mixin. + +This copy mechanism is limited to simple columns that have no foreign +keys, as a :class:`.ForeignKey` itself contains references to columns +which can't be properly recreated at this level. For columns that +have foreign keys, as well as for the variety of mapper-level constructs +that require destination-explicit context, the +:class:`~.declared_attr` decorator is provided so that +patterns common to many classes can be defined as callables:: + + from sqlalchemy.ext.declarative import declared_attr + + class ReferenceAddressMixin(object): + @declared_attr + def address_id(cls): + return Column(Integer, ForeignKey('address.id')) + + class User(ReferenceAddressMixin, Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + +Where above, the ``address_id`` class-level callable is executed at the +point at which the ``User`` class is constructed, and the declarative +extension can use the resulting :class:`.Column` object as returned by +the method without the need to copy it. + +.. versionchanged:: > 0.6.5 + Rename 0.6.5 ``sqlalchemy.util.classproperty`` + into :class:`~.declared_attr`. + +Columns generated by :class:`~.declared_attr` can also be +referenced by ``__mapper_args__`` to a limited degree, currently +by ``polymorphic_on`` and ``version_id_col``; the declarative extension +will resolve them at class construction time:: + + class MyMixin: + @declared_attr + def type_(cls): + return Column(String(50)) + + __mapper_args__= {'polymorphic_on':type_} + + class MyModel(MyMixin, Base): + __tablename__='test' + id = Column(Integer, primary_key=True) + + +Mixing in Relationships +~~~~~~~~~~~~~~~~~~~~~~~ + +Relationships created by :func:`~sqlalchemy.orm.relationship` are provided +with declarative mixin classes exclusively using the +:class:`.declared_attr` approach, eliminating any ambiguity +which could arise when copying a relationship and its possibly column-bound +contents. Below is an example which combines a foreign key column and a +relationship so that two classes ``Foo`` and ``Bar`` can both be configured to +reference a common target class via many-to-one:: + + class RefTargetMixin(object): + @declared_attr + def target_id(cls): + return Column('target_id', ForeignKey('target.id')) + + @declared_attr + def target(cls): + return relationship("Target") + + class Foo(RefTargetMixin, Base): + __tablename__ = 'foo' + id = Column(Integer, primary_key=True) + + class Bar(RefTargetMixin, Base): + __tablename__ = 'bar' + id = Column(Integer, primary_key=True) + + class Target(Base): + __tablename__ = 'target' + id = Column(Integer, primary_key=True) + + +Using Advanced Relationship Arguments (e.g. ``primaryjoin``, etc.) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`~sqlalchemy.orm.relationship` definitions which require explicit +primaryjoin, order_by etc. expressions should in all but the most +simplistic cases use **late bound** forms +for these arguments, meaning, using either the string form or a lambda. +The reason for this is that the related :class:`.Column` objects which are to +be configured using ``@declared_attr`` are not available to another +``@declared_attr`` attribute; while the methods will work and return new +:class:`.Column` objects, those are not the :class:`.Column` objects that +Declarative will be using as it calls the methods on its own, thus using +*different* :class:`.Column` objects. + +The canonical example is the primaryjoin condition that depends upon +another mixed-in column:: + + class RefTargetMixin(object): + @declared_attr + def target_id(cls): + return Column('target_id', ForeignKey('target.id')) + + @declared_attr + def target(cls): + return relationship(Target, + primaryjoin=Target.id==cls.target_id # this is *incorrect* + ) + +Mapping a class using the above mixin, we will get an error like:: + + sqlalchemy.exc.InvalidRequestError: this ForeignKey's parent column is not + yet associated with a Table. + +This is because the ``target_id`` :class:`.Column` we've called upon in our +``target()`` method is not the same :class:`.Column` that declarative is +actually going to map to our table. + +The condition above is resolved using a lambda:: + + class RefTargetMixin(object): + @declared_attr + def target_id(cls): + return Column('target_id', ForeignKey('target.id')) + + @declared_attr + def target(cls): + return relationship(Target, + primaryjoin=lambda: Target.id==cls.target_id + ) + +or alternatively, the string form (which ultimately generates a lambda):: + + class RefTargetMixin(object): + @declared_attr + def target_id(cls): + return Column('target_id', ForeignKey('target.id')) + + @declared_attr + def target(cls): + return relationship("Target", + primaryjoin="Target.id==%s.target_id" % cls.__name__ + ) + +Mixing in deferred(), column_property(), and other MapperProperty classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like :func:`~sqlalchemy.orm.relationship`, all +:class:`~sqlalchemy.orm.interfaces.MapperProperty` subclasses such as +:func:`~sqlalchemy.orm.deferred`, :func:`~sqlalchemy.orm.column_property`, +etc. ultimately involve references to columns, and therefore, when +used with declarative mixins, have the :class:`.declared_attr` +requirement so that no reliance on copying is needed:: + + class SomethingMixin(object): + + @declared_attr + def dprop(cls): + return deferred(Column(Integer)) + + class Something(SomethingMixin, Base): + __tablename__ = "something" + +The :func:`.column_property` or other construct may refer +to other columns from the mixin. These are copied ahead of time before +the :class:`.declared_attr` is invoked:: + + class SomethingMixin(object): + x = Column(Integer) + + y = Column(Integer) + + @declared_attr + def x_plus_y(cls): + return column_property(cls.x + cls.y) + + +.. versionchanged:: 1.0.0 mixin columns are copied to the final mapped class + so that :class:`.declared_attr` methods can access the actual column + that will be mapped. + +Mixing in Association Proxy and Other Attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Mixins can specify user-defined attributes as well as other extension +units such as :func:`.association_proxy`. The usage of +:class:`.declared_attr` is required in those cases where the attribute must +be tailored specifically to the target subclass. An example is when +constructing multiple :func:`.association_proxy` attributes which each +target a different type of child object. Below is an +:func:`.association_proxy` / mixin example which provides a scalar list of +string values to an implementing class:: + + from sqlalchemy import Column, Integer, ForeignKey, String + from sqlalchemy.orm import relationship + from sqlalchemy.ext.associationproxy import association_proxy + from sqlalchemy.ext.declarative import declarative_base, declared_attr + + Base = declarative_base() + + class HasStringCollection(object): + @declared_attr + def _strings(cls): + class StringAttribute(Base): + __tablename__ = cls.string_table_name + id = Column(Integer, primary_key=True) + value = Column(String(50), nullable=False) + parent_id = Column(Integer, + ForeignKey('%s.id' % cls.__tablename__), + nullable=False) + def __init__(self, value): + self.value = value + + return relationship(StringAttribute) + + @declared_attr + def strings(cls): + return association_proxy('_strings', 'value') + + class TypeA(HasStringCollection, Base): + __tablename__ = 'type_a' + string_table_name = 'type_a_strings' + id = Column(Integer(), primary_key=True) + + class TypeB(HasStringCollection, Base): + __tablename__ = 'type_b' + string_table_name = 'type_b_strings' + id = Column(Integer(), primary_key=True) + +Above, the ``HasStringCollection`` mixin produces a :func:`.relationship` +which refers to a newly generated class called ``StringAttribute``. The +``StringAttribute`` class is generated with its own :class:`.Table` +definition which is local to the parent class making usage of the +``HasStringCollection`` mixin. It also produces an :func:`.association_proxy` +object which proxies references to the ``strings`` attribute onto the ``value`` +attribute of each ``StringAttribute`` instance. + +``TypeA`` or ``TypeB`` can be instantiated given the constructor +argument ``strings``, a list of strings:: + + ta = TypeA(strings=['foo', 'bar']) + tb = TypeA(strings=['bat', 'bar']) + +This list will generate a collection +of ``StringAttribute`` objects, which are persisted into a table that's +local to either the ``type_a_strings`` or ``type_b_strings`` table:: + + >>> print ta._strings + [<__main__.StringAttribute object at 0x10151cd90>, + <__main__.StringAttribute object at 0x10151ce10>] + +When constructing the :func:`.association_proxy`, the +:class:`.declared_attr` decorator must be used so that a distinct +:func:`.association_proxy` object is created for each of the ``TypeA`` +and ``TypeB`` classes. + +.. versionadded:: 0.8 :class:`.declared_attr` is usable with non-mapped + attributes, including user-defined attributes as well as + :func:`.association_proxy`. + + +Controlling table inheritance with mixins +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``__tablename__`` attribute may be used to provide a function that +will determine the name of the table used for each class in an inheritance +hierarchy, as well as whether a class has its own distinct table. + +This is achieved using the :class:`.declared_attr` indicator in conjunction +with a method named ``__tablename__()``. Declarative will always +invoke :class:`.declared_attr` for the special names +``__tablename__``, ``__mapper_args__`` and ``__table_args__`` +function **for each mapped class in the hierarchy**. The function therefore +needs to expect to receive each class individually and to provide the +correct answer for each. + +For example, to create a mixin that gives every class a simple table +name based on class name:: + + from sqlalchemy.ext.declarative import declared_attr + + class Tablename: + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() + + class Person(Tablename, Base): + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __tablename__ = None + __mapper_args__ = {'polymorphic_identity': 'engineer'} + primary_language = Column(String(50)) + +Alternatively, we can modify our ``__tablename__`` function to return +``None`` for subclasses, using :func:`.has_inherited_table`. This has +the effect of those subclasses being mapped with single table inheritance +agaisnt the parent:: + + from sqlalchemy.ext.declarative import declared_attr + from sqlalchemy.ext.declarative import has_inherited_table + + class Tablename(object): + @declared_attr + def __tablename__(cls): + if has_inherited_table(cls): + return None + return cls.__name__.lower() + + class Person(Tablename, Base): + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + primary_language = Column(String(50)) + __mapper_args__ = {'polymorphic_identity': 'engineer'} + +.. _mixin_inheritance_columns: + +Mixing in Columns in Inheritance Scenarios +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In constrast to how ``__tablename__`` and other special names are handled when +used with :class:`.declared_attr`, when we mix in columns and properties (e.g. +relationships, column properties, etc.), the function is +invoked for the **base class only** in the hierarchy. Below, only the +``Person`` class will receive a column +called ``id``; the mapping will fail on ``Engineer``, which is not given +a primary key:: + + class HasId(object): + @declared_attr + def id(cls): + return Column('id', Integer, primary_key=True) + + class Person(HasId, Base): + __tablename__ = 'person' + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __tablename__ = 'engineer' + primary_language = Column(String(50)) + __mapper_args__ = {'polymorphic_identity': 'engineer'} + +It is usually the case in joined-table inheritance that we want distinctly +named columns on each subclass. However in this case, we may want to have +an ``id`` column on every table, and have them refer to each other via +foreign key. We can achieve this as a mixin by using the +:attr:`.declared_attr.cascading` modifier, which indicates that the +function should be invoked **for each class in the hierarchy**, just like +it does for ``__tablename__``:: + + class HasId(object): + @declared_attr.cascading + def id(cls): + if has_inherited_table(cls): + return Column('id', + Integer, + ForeignKey('person.id'), primary_key=True) + else: + return Column('id', Integer, primary_key=True) + + class Person(HasId, Base): + __tablename__ = 'person' + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __tablename__ = 'engineer' + primary_language = Column(String(50)) + __mapper_args__ = {'polymorphic_identity': 'engineer'} + + +.. versionadded:: 1.0.0 added :attr:`.declared_attr.cascading`. + +Combining Table/Mapper Arguments from Multiple Mixins +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the case of ``__table_args__`` or ``__mapper_args__`` +specified with declarative mixins, you may want to combine +some parameters from several mixins with those you wish to +define on the class iteself. The +:class:`.declared_attr` decorator can be used +here to create user-defined collation routines that pull +from multiple collections:: + + from sqlalchemy.ext.declarative import declared_attr + + class MySQLSettings(object): + __table_args__ = {'mysql_engine':'InnoDB'} + + class MyOtherMixin(object): + __table_args__ = {'info':'foo'} + + class MyModel(MySQLSettings, MyOtherMixin, Base): + __tablename__='my_model' + + @declared_attr + def __table_args__(cls): + args = dict() + args.update(MySQLSettings.__table_args__) + args.update(MyOtherMixin.__table_args__) + return args + + id = Column(Integer, primary_key=True) + +Creating Indexes with Mixins +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To define a named, potentially multicolumn :class:`.Index` that applies to all +tables derived from a mixin, use the "inline" form of :class:`.Index` and +establish it as part of ``__table_args__``:: + + class MyMixin(object): + a = Column(Integer) + b = Column(Integer) + + @declared_attr + def __table_args__(cls): + return (Index('test_idx_%s' % cls.__tablename__, 'a', 'b'),) + + class MyModel(MyMixin, Base): + __tablename__ = 'atable' + c = Column(Integer,primary_key=True) diff --git a/doc/build/orm/extensions/declarative/relationships.rst b/doc/build/orm/extensions/declarative/relationships.rst new file mode 100644 index 000000000..fb53c28bb --- /dev/null +++ b/doc/build/orm/extensions/declarative/relationships.rst @@ -0,0 +1,138 @@ +.. _declarative_configuring_relationships: + +========================= +Configuring Relationships +========================= + +Relationships to other classes are done in the usual way, with the added +feature that the class specified to :func:`~sqlalchemy.orm.relationship` +may be a string name. The "class registry" associated with ``Base`` +is used at mapper compilation time to resolve the name into the actual +class object, which is expected to have been defined once the mapper +configuration is used:: + + class User(Base): + __tablename__ = 'users' + + id = Column(Integer, primary_key=True) + name = Column(String(50)) + addresses = relationship("Address", backref="user") + + class Address(Base): + __tablename__ = 'addresses' + + id = Column(Integer, primary_key=True) + email = Column(String(50)) + user_id = Column(Integer, ForeignKey('users.id')) + +Column constructs, since they are just that, are immediately usable, +as below where we define a primary join condition on the ``Address`` +class using them:: + + class Address(Base): + __tablename__ = 'addresses' + + id = Column(Integer, primary_key=True) + email = Column(String(50)) + user_id = Column(Integer, ForeignKey('users.id')) + user = relationship(User, primaryjoin=user_id == User.id) + +In addition to the main argument for :func:`~sqlalchemy.orm.relationship`, +other arguments which depend upon the columns present on an as-yet +undefined class may also be specified as strings. These strings are +evaluated as Python expressions. The full namespace available within +this evaluation includes all classes mapped for this declarative base, +as well as the contents of the ``sqlalchemy`` package, including +expression functions like :func:`~sqlalchemy.sql.expression.desc` and +:attr:`~sqlalchemy.sql.expression.func`:: + + class User(Base): + # .... + addresses = relationship("Address", + order_by="desc(Address.email)", + primaryjoin="Address.user_id==User.id") + +For the case where more than one module contains a class of the same name, +string class names can also be specified as module-qualified paths +within any of these string expressions:: + + class User(Base): + # .... + addresses = relationship("myapp.model.address.Address", + order_by="desc(myapp.model.address.Address.email)", + primaryjoin="myapp.model.address.Address.user_id==" + "myapp.model.user.User.id") + +The qualified path can be any partial path that removes ambiguity between +the names. For example, to disambiguate between +``myapp.model.address.Address`` and ``myapp.model.lookup.Address``, +we can specify ``address.Address`` or ``lookup.Address``:: + + class User(Base): + # .... + addresses = relationship("address.Address", + order_by="desc(address.Address.email)", + primaryjoin="address.Address.user_id==" + "User.id") + +.. versionadded:: 0.8 + module-qualified paths can be used when specifying string arguments + with Declarative, in order to specify specific modules. + +Two alternatives also exist to using string-based attributes. A lambda +can also be used, which will be evaluated after all mappers have been +configured:: + + class User(Base): + # ... + addresses = relationship(lambda: Address, + order_by=lambda: desc(Address.email), + primaryjoin=lambda: Address.user_id==User.id) + +Or, the relationship can be added to the class explicitly after the classes +are available:: + + User.addresses = relationship(Address, + primaryjoin=Address.user_id==User.id) + + + +.. _declarative_many_to_many: + +Configuring Many-to-Many Relationships +====================================== + +Many-to-many relationships are also declared in the same way +with declarative as with traditional mappings. The +``secondary`` argument to +:func:`.relationship` is as usual passed a +:class:`.Table` object, which is typically declared in the +traditional way. The :class:`.Table` usually shares +the :class:`.MetaData` object used by the declarative base:: + + keywords = Table( + 'keywords', Base.metadata, + Column('author_id', Integer, ForeignKey('authors.id')), + Column('keyword_id', Integer, ForeignKey('keywords.id')) + ) + + class Author(Base): + __tablename__ = 'authors' + id = Column(Integer, primary_key=True) + keywords = relationship("Keyword", secondary=keywords) + +Like other :func:`~sqlalchemy.orm.relationship` arguments, a string is accepted +as well, passing the string name of the table as defined in the +``Base.metadata.tables`` collection:: + + class Author(Base): + __tablename__ = 'authors' + id = Column(Integer, primary_key=True) + keywords = relationship("Keyword", secondary="keywords") + +As with traditional mapping, its generally not a good idea to use +a :class:`.Table` as the "secondary" argument which is also mapped to +a class, unless the :func:`.relationship` is declared with ``viewonly=True``. +Otherwise, the unit-of-work system may attempt duplicate INSERT and +DELETE statements against the underlying table. + diff --git a/doc/build/orm/extensions/declarative/table_config.rst b/doc/build/orm/extensions/declarative/table_config.rst new file mode 100644 index 000000000..9a621e6dd --- /dev/null +++ b/doc/build/orm/extensions/declarative/table_config.rst @@ -0,0 +1,143 @@ +.. _declarative_table_args: + +=================== +Table Configuration +=================== + +Table arguments other than the name, metadata, and mapped Column +arguments are specified using the ``__table_args__`` class attribute. +This attribute accommodates both positional as well as keyword +arguments that are normally sent to the +:class:`~sqlalchemy.schema.Table` constructor. +The attribute can be specified in one of two forms. One is as a +dictionary:: + + class MyClass(Base): + __tablename__ = 'sometable' + __table_args__ = {'mysql_engine':'InnoDB'} + +The other, a tuple, where each argument is positional +(usually constraints):: + + class MyClass(Base): + __tablename__ = 'sometable' + __table_args__ = ( + ForeignKeyConstraint(['id'], ['remote_table.id']), + UniqueConstraint('foo'), + ) + +Keyword arguments can be specified with the above form by +specifying the last argument as a dictionary:: + + class MyClass(Base): + __tablename__ = 'sometable' + __table_args__ = ( + ForeignKeyConstraint(['id'], ['remote_table.id']), + UniqueConstraint('foo'), + {'autoload':True} + ) + +Using a Hybrid Approach with __table__ +======================================= + +As an alternative to ``__tablename__``, a direct +:class:`~sqlalchemy.schema.Table` construct may be used. The +:class:`~sqlalchemy.schema.Column` objects, which in this case require +their names, will be added to the mapping just like a regular mapping +to a table:: + + class MyClass(Base): + __table__ = Table('my_table', Base.metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)) + ) + +``__table__`` provides a more focused point of control for establishing +table metadata, while still getting most of the benefits of using declarative. +An application that uses reflection might want to load table metadata elsewhere +and pass it to declarative classes:: + + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + Base.metadata.reflect(some_engine) + + class User(Base): + __table__ = metadata.tables['user'] + + class Address(Base): + __table__ = metadata.tables['address'] + +Some configuration schemes may find it more appropriate to use ``__table__``, +such as those which already take advantage of the data-driven nature of +:class:`.Table` to customize and/or automate schema definition. + +Note that when the ``__table__`` approach is used, the object is immediately +usable as a plain :class:`.Table` within the class declaration body itself, +as a Python class is only another syntactical block. Below this is illustrated +by using the ``id`` column in the ``primaryjoin`` condition of a +:func:`.relationship`:: + + class MyClass(Base): + __table__ = Table('my_table', Base.metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)) + ) + + widgets = relationship(Widget, + primaryjoin=Widget.myclass_id==__table__.c.id) + +Similarly, mapped attributes which refer to ``__table__`` can be placed inline, +as below where we assign the ``name`` column to the attribute ``_name``, +generating a synonym for ``name``:: + + from sqlalchemy.ext.declarative import synonym_for + + class MyClass(Base): + __table__ = Table('my_table', Base.metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)) + ) + + _name = __table__.c.name + + @synonym_for("_name") + def name(self): + return "Name: %s" % _name + +Using Reflection with Declarative +================================= + +It's easy to set up a :class:`.Table` that uses ``autoload=True`` +in conjunction with a mapped class:: + + class MyClass(Base): + __table__ = Table('mytable', Base.metadata, + autoload=True, autoload_with=some_engine) + +However, one improvement that can be made here is to not +require the :class:`.Engine` to be available when classes are +being first declared. To achieve this, use the +:class:`.DeferredReflection` mixin, which sets up mappings +only after a special ``prepare(engine)`` step is called:: + + from sqlalchemy.ext.declarative import declarative_base, DeferredReflection + + Base = declarative_base(cls=DeferredReflection) + + class Foo(Base): + __tablename__ = 'foo' + bars = relationship("Bar") + + class Bar(Base): + __tablename__ = 'bar' + + # illustrate overriding of "bar.foo_id" to have + # a foreign key constraint otherwise not + # reflected, such as when using MySQL + foo_id = Column(Integer, ForeignKey('foo.id')) + + Base.prepare(e) + +.. versionadded:: 0.8 + Added :class:`.DeferredReflection`. diff --git a/doc/build/orm/extensions/index.rst b/doc/build/orm/extensions/index.rst index 65836f13a..f7f58e381 100644 --- a/doc/build/orm/extensions/index.rst +++ b/doc/build/orm/extensions/index.rst @@ -17,7 +17,7 @@ behavior. In particular the "Horizontal Sharding", "Hybrid Attributes", and associationproxy automap - declarative + declarative/index mutable orderinglist horizontal_shard diff --git a/doc/build/orm/index.rst b/doc/build/orm/index.rst index 6c12ebd38..b7683a8ad 100644 --- a/doc/build/orm/index.rst +++ b/doc/build/orm/index.rst @@ -9,18 +9,13 @@ as well as automated persistence of Python objects, proceed first to the tutorial. .. toctree:: - :maxdepth: 3 + :maxdepth: 2 tutorial mapper_config relationships - collections - inheritance + loading_objects session - query - loading - events + extending extensions/index examples - exceptions - internals diff --git a/doc/build/orm/join_conditions.rst b/doc/build/orm/join_conditions.rst new file mode 100644 index 000000000..5e2c11d1d --- /dev/null +++ b/doc/build/orm/join_conditions.rst @@ -0,0 +1,740 @@ +.. _relationship_configure_joins: + +Configuring how Relationship Joins +------------------------------------ + +:func:`.relationship` will normally create a join between two tables +by examining the foreign key relationship between the two tables +to determine which columns should be compared. There are a variety +of situations where this behavior needs to be customized. + +.. _relationship_foreign_keys: + +Handling Multiple Join Paths +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One of the most common situations to deal with is when +there are more than one foreign key path between two tables. + +Consider a ``Customer`` class that contains two foreign keys to an ``Address`` +class:: + + from sqlalchemy import Integer, ForeignKey, String, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class Customer(Base): + __tablename__ = 'customer' + id = Column(Integer, primary_key=True) + name = Column(String) + + billing_address_id = Column(Integer, ForeignKey("address.id")) + shipping_address_id = Column(Integer, ForeignKey("address.id")) + + billing_address = relationship("Address") + shipping_address = relationship("Address") + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + street = Column(String) + city = Column(String) + state = Column(String) + zip = Column(String) + +The above mapping, when we attempt to use it, will produce the error:: + + sqlalchemy.exc.AmbiguousForeignKeysError: Could not determine join + condition between parent/child tables on relationship + Customer.billing_address - there are multiple foreign key + paths linking the tables. Specify the 'foreign_keys' argument, + providing a list of those columns which should be + counted as containing a foreign key reference to the parent table. + +The above message is pretty long. There are many potential messages +that :func:`.relationship` can return, which have been carefully tailored +to detect a variety of common configurational issues; most will suggest +the additional configuration that's needed to resolve the ambiguity +or other missing information. + +In this case, the message wants us to qualify each :func:`.relationship` +by instructing for each one which foreign key column should be considered, and +the appropriate form is as follows:: + + class Customer(Base): + __tablename__ = 'customer' + id = Column(Integer, primary_key=True) + name = Column(String) + + billing_address_id = Column(Integer, ForeignKey("address.id")) + shipping_address_id = Column(Integer, ForeignKey("address.id")) + + billing_address = relationship("Address", foreign_keys=[billing_address_id]) + shipping_address = relationship("Address", foreign_keys=[shipping_address_id]) + +Above, we specify the ``foreign_keys`` argument, which is a :class:`.Column` or list +of :class:`.Column` objects which indicate those columns to be considered "foreign", +or in other words, the columns that contain a value referring to a parent table. +Loading the ``Customer.billing_address`` relationship from a ``Customer`` +object will use the value present in ``billing_address_id`` in order to +identify the row in ``Address`` to be loaded; similarly, ``shipping_address_id`` +is used for the ``shipping_address`` relationship. The linkage of the two +columns also plays a role during persistence; the newly generated primary key +of a just-inserted ``Address`` object will be copied into the appropriate +foreign key column of an associated ``Customer`` object during a flush. + +When specifying ``foreign_keys`` with Declarative, we can also use string +names to specify, however it is important that if using a list, the **list +is part of the string**:: + + billing_address = relationship("Address", foreign_keys="[Customer.billing_address_id]") + +In this specific example, the list is not necessary in any case as there's only +one :class:`.Column` we need:: + + billing_address = relationship("Address", foreign_keys="Customer.billing_address_id") + +.. versionchanged:: 0.8 + :func:`.relationship` can resolve ambiguity between foreign key targets on the + basis of the ``foreign_keys`` argument alone; the :paramref:`~.relationship.primaryjoin` + argument is no longer needed in this situation. + +.. _relationship_primaryjoin: + +Specifying Alternate Join Conditions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The default behavior of :func:`.relationship` when constructing a join +is that it equates the value of primary key columns +on one side to that of foreign-key-referring columns on the other. +We can change this criterion to be anything we'd like using the +:paramref:`~.relationship.primaryjoin` +argument, as well as the :paramref:`~.relationship.secondaryjoin` +argument in the case when a "secondary" table is used. + +In the example below, using the ``User`` class +as well as an ``Address`` class which stores a street address, we +create a relationship ``boston_addresses`` which will only +load those ``Address`` objects which specify a city of "Boston":: + + from sqlalchemy import Integer, ForeignKey, String, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + name = Column(String) + boston_addresses = relationship("Address", + primaryjoin="and_(User.id==Address.user_id, " + "Address.city=='Boston')") + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('user.id')) + + street = Column(String) + city = Column(String) + state = Column(String) + zip = Column(String) + +Within this string SQL expression, we made use of the :func:`.and_` conjunction construct to establish +two distinct predicates for the join condition - joining both the ``User.id`` and +``Address.user_id`` columns to each other, as well as limiting rows in ``Address`` +to just ``city='Boston'``. When using Declarative, rudimentary SQL functions like +:func:`.and_` are automatically available in the evaluated namespace of a string +:func:`.relationship` argument. + +The custom criteria we use in a :paramref:`~.relationship.primaryjoin` +is generally only significant when SQLAlchemy is rendering SQL in +order to load or represent this relationship. That is, it's used in +the SQL statement that's emitted in order to perform a per-attribute +lazy load, or when a join is constructed at query time, such as via +:meth:`.Query.join`, or via the eager "joined" or "subquery" styles of +loading. When in-memory objects are being manipulated, we can place +any ``Address`` object we'd like into the ``boston_addresses`` +collection, regardless of what the value of the ``.city`` attribute +is. The objects will remain present in the collection until the +attribute is expired and re-loaded from the database where the +criterion is applied. When a flush occurs, the objects inside of +``boston_addresses`` will be flushed unconditionally, assigning value +of the primary key ``user.id`` column onto the foreign-key-holding +``address.user_id`` column for each row. The ``city`` criteria has no +effect here, as the flush process only cares about synchronizing +primary key values into referencing foreign key values. + +.. _relationship_custom_foreign: + +Creating Custom Foreign Conditions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Another element of the primary join condition is how those columns +considered "foreign" are determined. Usually, some subset +of :class:`.Column` objects will specify :class:`.ForeignKey`, or otherwise +be part of a :class:`.ForeignKeyConstraint` that's relevant to the join condition. +:func:`.relationship` looks to this foreign key status as it decides +how it should load and persist data for this relationship. However, the +:paramref:`~.relationship.primaryjoin` argument can be used to create a join condition that +doesn't involve any "schema" level foreign keys. We can combine :paramref:`~.relationship.primaryjoin` +along with :paramref:`~.relationship.foreign_keys` and :paramref:`~.relationship.remote_side` explicitly in order to +establish such a join. + +Below, a class ``HostEntry`` joins to itself, equating the string ``content`` +column to the ``ip_address`` column, which is a Postgresql type called ``INET``. +We need to use :func:`.cast` in order to cast one side of the join to the +type of the other:: + + from sqlalchemy import cast, String, Column, Integer + from sqlalchemy.orm import relationship + from sqlalchemy.dialects.postgresql import INET + + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + class HostEntry(Base): + __tablename__ = 'host_entry' + + id = Column(Integer, primary_key=True) + ip_address = Column(INET) + content = Column(String(50)) + + # relationship() using explicit foreign_keys, remote_side + parent_host = relationship("HostEntry", + primaryjoin=ip_address == cast(content, INET), + foreign_keys=content, + remote_side=ip_address + ) + +The above relationship will produce a join like:: + + SELECT host_entry.id, host_entry.ip_address, host_entry.content + FROM host_entry JOIN host_entry AS host_entry_1 + ON host_entry_1.ip_address = CAST(host_entry.content AS INET) + +An alternative syntax to the above is to use the :func:`.foreign` and +:func:`.remote` :term:`annotations`, +inline within the :paramref:`~.relationship.primaryjoin` expression. +This syntax represents the annotations that :func:`.relationship` normally +applies by itself to the join condition given the :paramref:`~.relationship.foreign_keys` and +:paramref:`~.relationship.remote_side` arguments. These functions may +be more succinct when an explicit join condition is present, and additionally +serve to mark exactly the column that is "foreign" or "remote" independent +of whether that column is stated multiple times or within complex +SQL expressions:: + + from sqlalchemy.orm import foreign, remote + + class HostEntry(Base): + __tablename__ = 'host_entry' + + id = Column(Integer, primary_key=True) + ip_address = Column(INET) + content = Column(String(50)) + + # relationship() using explicit foreign() and remote() annotations + # in lieu of separate arguments + parent_host = relationship("HostEntry", + primaryjoin=remote(ip_address) == \ + cast(foreign(content), INET), + ) + + +.. _relationship_custom_operator: + +Using custom operators in join conditions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Another use case for relationships is the use of custom operators, such +as Postgresql's "is contained within" ``<<`` operator when joining with +types such as :class:`.postgresql.INET` and :class:`.postgresql.CIDR`. +For custom operators we use the :meth:`.Operators.op` function:: + + inet_column.op("<<")(cidr_column) + +However, if we construct a :paramref:`~.relationship.primaryjoin` using this +operator, :func:`.relationship` will still need more information. This is because +when it examines our primaryjoin condition, it specifically looks for operators +used for **comparisons**, and this is typically a fixed list containing known +comparison operators such as ``==``, ``<``, etc. So for our custom operator +to participate in this system, we need it to register as a comparison operator +using the :paramref:`~.Operators.op.is_comparison` parameter:: + + inet_column.op("<<", is_comparison=True)(cidr_column) + +A complete example:: + + class IPA(Base): + __tablename__ = 'ip_address' + + id = Column(Integer, primary_key=True) + v4address = Column(INET) + + network = relationship("Network", + primaryjoin="IPA.v4address.op('<<', is_comparison=True)" + "(foreign(Network.v4representation))", + viewonly=True + ) + class Network(Base): + __tablename__ = 'network' + + id = Column(Integer, primary_key=True) + v4representation = Column(CIDR) + +Above, a query such as:: + + session.query(IPA).join(IPA.network) + +Will render as:: + + SELECT ip_address.id AS ip_address_id, ip_address.v4address AS ip_address_v4address + FROM ip_address JOIN network ON ip_address.v4address << network.v4representation + +.. versionadded:: 0.9.2 - Added the :paramref:`.Operators.op.is_comparison` + flag to assist in the creation of :func:`.relationship` constructs using + custom operators. + +.. _relationship_overlapping_foreignkeys: + +Overlapping Foreign Keys +~~~~~~~~~~~~~~~~~~~~~~~~ + +A rare scenario can arise when composite foreign keys are used, such that +a single column may be the subject of more than one column +referred to via foreign key constraint. + +Consider an (admittedly complex) mapping such as the ``Magazine`` object, +referred to both by the ``Writer`` object and the ``Article`` object +using a composite primary key scheme that includes ``magazine_id`` +for both; then to make ``Article`` refer to ``Writer`` as well, +``Article.magazine_id`` is involved in two separate relationships; +``Article.magazine`` and ``Article.writer``:: + + class Magazine(Base): + __tablename__ = 'magazine' + + id = Column(Integer, primary_key=True) + + + class Article(Base): + __tablename__ = 'article' + + article_id = Column(Integer) + magazine_id = Column(ForeignKey('magazine.id')) + writer_id = Column() + + magazine = relationship("Magazine") + writer = relationship("Writer") + + __table_args__ = ( + PrimaryKeyConstraint('article_id', 'magazine_id'), + ForeignKeyConstraint( + ['writer_id', 'magazine_id'], + ['writer.id', 'writer.magazine_id'] + ), + ) + + + class Writer(Base): + __tablename__ = 'writer' + + id = Column(Integer, primary_key=True) + magazine_id = Column(ForeignKey('magazine.id'), primary_key=True) + magazine = relationship("Magazine") + +When the above mapping is configured, we will see this warning emitted:: + + SAWarning: relationship 'Article.writer' will copy column + writer.magazine_id to column article.magazine_id, + which conflicts with relationship(s): 'Article.magazine' + (copies magazine.id to article.magazine_id). Consider applying + viewonly=True to read-only relationships, or provide a primaryjoin + condition marking writable columns with the foreign() annotation. + +What this refers to originates from the fact that ``Article.magazine_id`` is +the subject of two different foreign key constraints; it refers to +``Magazine.id`` directly as a source column, but also refers to +``Writer.magazine_id`` as a source column in the context of the +composite key to ``Writer``. If we associate an ``Article`` with a +particular ``Magazine``, but then associate the ``Article`` with a +``Writer`` that's associated with a *different* ``Magazine``, the ORM +will overwrite ``Article.magazine_id`` non-deterministically, silently +changing which magazine we refer towards; it may +also attempt to place NULL into this columnn if we de-associate a +``Writer`` from an ``Article``. The warning lets us know this is the case. + +To solve this, we need to break out the behavior of ``Article`` to include +all three of the following features: + +1. ``Article`` first and foremost writes to + ``Article.magazine_id`` based on data persisted in the ``Article.magazine`` + relationship only, that is a value copied from ``Magazine.id``. + +2. ``Article`` can write to ``Article.writer_id`` on behalf of data + persisted in the ``Article.writer`` relationship, but only the + ``Writer.id`` column; the ``Writer.magazine_id`` column should not + be written into ``Article.magazine_id`` as it ultimately is sourced + from ``Magazine.id``. + +3. ``Article`` takes ``Article.magazine_id`` into account when loading + ``Article.writer``, even though it *doesn't* write to it on behalf + of this relationship. + +To get just #1 and #2, we could specify only ``Article.writer_id`` as the +"foreign keys" for ``Article.writer``:: + + class Article(Base): + # ... + + writer = relationship("Writer", foreign_keys='Article.writer_id') + +However, this has the effect of ``Article.writer`` not taking +``Article.magazine_id`` into account when querying against ``Writer``: + +.. sourcecode:: sql + + SELECT article.article_id AS article_article_id, + article.magazine_id AS article_magazine_id, + article.writer_id AS article_writer_id + FROM article + JOIN writer ON writer.id = article.writer_id + +Therefore, to get at all of #1, #2, and #3, we express the join condition +as well as which columns to be written by combining +:paramref:`~.relationship.primaryjoin` fully, along with either the +:paramref:`~.relationship.foreign_keys` argument, or more succinctly by +annotating with :func:`~.orm.foreign`:: + + class Article(Base): + # ... + + writer = relationship( + "Writer", + primaryjoin="and_(Writer.id == foreign(Article.writer_id), " + "Writer.magazine_id == Article.magazine_id)") + +.. versionchanged:: 1.0.0 the ORM will attempt to warn when a column is used + as the synchronization target from more than one relationship + simultaneously. + + +Non-relational Comparisons / Materialized Path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. warning:: this section details an experimental feature. + +Using custom expressions means we can produce unorthodox join conditions that +don't obey the usual primary/foreign key model. One such example is the +materialized path pattern, where we compare strings for overlapping path tokens +in order to produce a tree structure. + +Through careful use of :func:`.foreign` and :func:`.remote`, we can build +a relationship that effectively produces a rudimentary materialized path +system. Essentially, when :func:`.foreign` and :func:`.remote` are +on the *same* side of the comparison expression, the relationship is considered +to be "one to many"; when they are on *different* sides, the relationship +is considered to be "many to one". For the comparison we'll use here, +we'll be dealing with collections so we keep things configured as "one to many":: + + class Element(Base): + __tablename__ = 'element' + + path = Column(String, primary_key=True) + + descendants = relationship('Element', + primaryjoin= + remote(foreign(path)).like( + path.concat('/%')), + viewonly=True, + order_by=path) + +Above, if given an ``Element`` object with a path attribute of ``"/foo/bar2"``, +we seek for a load of ``Element.descendants`` to look like:: + + SELECT element.path AS element_path + FROM element + WHERE element.path LIKE ('/foo/bar2' || '/%') ORDER BY element.path + +.. versionadded:: 0.9.5 Support has been added to allow a single-column + comparison to itself within a primaryjoin condition, as well as for + primaryjoin conditions that use :meth:`.Operators.like` as the comparison + operator. + +.. _self_referential_many_to_many: + +Self-Referential Many-to-Many Relationship +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many to many relationships can be customized by one or both of :paramref:`~.relationship.primaryjoin` +and :paramref:`~.relationship.secondaryjoin` - the latter is significant for a relationship that +specifies a many-to-many reference using the :paramref:`~.relationship.secondary` argument. +A common situation which involves the usage of :paramref:`~.relationship.primaryjoin` and :paramref:`~.relationship.secondaryjoin` +is when establishing a many-to-many relationship from a class to itself, as shown below:: + + from sqlalchemy import Integer, ForeignKey, String, Column, Table + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + node_to_node = Table("node_to_node", Base.metadata, + Column("left_node_id", Integer, ForeignKey("node.id"), primary_key=True), + Column("right_node_id", Integer, ForeignKey("node.id"), primary_key=True) + ) + + class Node(Base): + __tablename__ = 'node' + id = Column(Integer, primary_key=True) + label = Column(String) + right_nodes = relationship("Node", + secondary=node_to_node, + primaryjoin=id==node_to_node.c.left_node_id, + secondaryjoin=id==node_to_node.c.right_node_id, + backref="left_nodes" + ) + +Where above, SQLAlchemy can't know automatically which columns should connect +to which for the ``right_nodes`` and ``left_nodes`` relationships. The :paramref:`~.relationship.primaryjoin` +and :paramref:`~.relationship.secondaryjoin` arguments establish how we'd like to join to the association table. +In the Declarative form above, as we are declaring these conditions within the Python +block that corresponds to the ``Node`` class, the ``id`` variable is available directly +as the :class:`.Column` object we wish to join with. + +Alternatively, we can define the :paramref:`~.relationship.primaryjoin` +and :paramref:`~.relationship.secondaryjoin` arguments using strings, which is suitable +in the case that our configuration does not have either the ``Node.id`` column +object available yet or the ``node_to_node`` table perhaps isn't yet available. +When referring to a plain :class:`.Table` object in a declarative string, we +use the string name of the table as it is present in the :class:`.MetaData`:: + + class Node(Base): + __tablename__ = 'node' + id = Column(Integer, primary_key=True) + label = Column(String) + right_nodes = relationship("Node", + secondary="node_to_node", + primaryjoin="Node.id==node_to_node.c.left_node_id", + secondaryjoin="Node.id==node_to_node.c.right_node_id", + backref="left_nodes" + ) + +A classical mapping situation here is similar, where ``node_to_node`` can be joined +to ``node.c.id``:: + + from sqlalchemy import Integer, ForeignKey, String, Column, Table, MetaData + from sqlalchemy.orm import relationship, mapper + + metadata = MetaData() + + node_to_node = Table("node_to_node", metadata, + Column("left_node_id", Integer, ForeignKey("node.id"), primary_key=True), + Column("right_node_id", Integer, ForeignKey("node.id"), primary_key=True) + ) + + node = Table("node", metadata, + Column('id', Integer, primary_key=True), + Column('label', String) + ) + class Node(object): + pass + + mapper(Node, node, properties={ + 'right_nodes':relationship(Node, + secondary=node_to_node, + primaryjoin=node.c.id==node_to_node.c.left_node_id, + secondaryjoin=node.c.id==node_to_node.c.right_node_id, + backref="left_nodes" + )}) + + +Note that in both examples, the :paramref:`~.relationship.backref` +keyword specifies a ``left_nodes`` backref - when +:func:`.relationship` creates the second relationship in the reverse +direction, it's smart enough to reverse the +:paramref:`~.relationship.primaryjoin` and +:paramref:`~.relationship.secondaryjoin` arguments. + +.. _composite_secondary_join: + +Composite "Secondary" Joins +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + This section features some new and experimental features of SQLAlchemy. + +Sometimes, when one seeks to build a :func:`.relationship` between two tables +there is a need for more than just two or three tables to be involved in +order to join them. This is an area of :func:`.relationship` where one seeks +to push the boundaries of what's possible, and often the ultimate solution to +many of these exotic use cases needs to be hammered out on the SQLAlchemy mailing +list. + +In more recent versions of SQLAlchemy, the :paramref:`~.relationship.secondary` +parameter can be used in some of these cases in order to provide a composite +target consisting of multiple tables. Below is an example of such a +join condition (requires version 0.9.2 at least to function as is):: + + class A(Base): + __tablename__ = 'a' + + id = Column(Integer, primary_key=True) + b_id = Column(ForeignKey('b.id')) + + d = relationship("D", + secondary="join(B, D, B.d_id == D.id)." + "join(C, C.d_id == D.id)", + primaryjoin="and_(A.b_id == B.id, A.id == C.a_id)", + secondaryjoin="D.id == B.d_id", + uselist=False + ) + + class B(Base): + __tablename__ = 'b' + + id = Column(Integer, primary_key=True) + d_id = Column(ForeignKey('d.id')) + + class C(Base): + __tablename__ = 'c' + + id = Column(Integer, primary_key=True) + a_id = Column(ForeignKey('a.id')) + d_id = Column(ForeignKey('d.id')) + + class D(Base): + __tablename__ = 'd' + + id = Column(Integer, primary_key=True) + +In the above example, we provide all three of :paramref:`~.relationship.secondary`, +:paramref:`~.relationship.primaryjoin`, and :paramref:`~.relationship.secondaryjoin`, +in the declarative style referring to the named tables ``a``, ``b``, ``c``, ``d`` +directly. A query from ``A`` to ``D`` looks like: + +.. sourcecode:: python+sql + + sess.query(A).join(A.d).all() + + {opensql}SELECT a.id AS a_id, a.b_id AS a_b_id + FROM a JOIN ( + b AS b_1 JOIN d AS d_1 ON b_1.d_id = d_1.id + JOIN c AS c_1 ON c_1.d_id = d_1.id) + ON a.b_id = b_1.id AND a.id = c_1.a_id JOIN d ON d.id = b_1.d_id + +In the above example, we take advantage of being able to stuff multiple +tables into a "secondary" container, so that we can join across many +tables while still keeping things "simple" for :func:`.relationship`, in that +there's just "one" table on both the "left" and the "right" side; the +complexity is kept within the middle. + +.. versionadded:: 0.9.2 Support is improved for allowing a :func:`.join()` + construct to be used directly as the target of the :paramref:`~.relationship.secondary` + argument, including support for joins, eager joins and lazy loading, + as well as support within declarative to specify complex conditions such + as joins involving class names as targets. + +.. _relationship_non_primary_mapper: + +Relationship to Non Primary Mapper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the previous section, we illustrated a technique where we used +:paramref:`~.relationship.secondary` in order to place additional +tables within a join condition. There is one complex join case where +even this technique is not sufficient; when we seek to join from ``A`` +to ``B``, making use of any number of ``C``, ``D``, etc. in between, +however there are also join conditions between ``A`` and ``B`` +*directly*. In this case, the join from ``A`` to ``B`` may be +difficult to express with just a complex +:paramref:`~.relationship.primaryjoin` condition, as the intermediary +tables may need special handling, and it is also not expressable with +a :paramref:`~.relationship.secondary` object, since the +``A->secondary->B`` pattern does not support any references between +``A`` and ``B`` directly. When this **extremely advanced** case +arises, we can resort to creating a second mapping as a target for the +relationship. This is where we use :func:`.mapper` in order to make a +mapping to a class that includes all the additional tables we need for +this join. In order to produce this mapper as an "alternative" mapping +for our class, we use the :paramref:`~.mapper.non_primary` flag. + +Below illustrates a :func:`.relationship` with a simple join from ``A`` to +``B``, however the primaryjoin condition is augmented with two additional +entities ``C`` and ``D``, which also must have rows that line up with +the rows in both ``A`` and ``B`` simultaneously:: + + class A(Base): + __tablename__ = 'a' + + id = Column(Integer, primary_key=True) + b_id = Column(ForeignKey('b.id')) + + class B(Base): + __tablename__ = 'b' + + id = Column(Integer, primary_key=True) + + class C(Base): + __tablename__ = 'c' + + id = Column(Integer, primary_key=True) + a_id = Column(ForeignKey('a.id')) + + class D(Base): + __tablename__ = 'd' + + id = Column(Integer, primary_key=True) + c_id = Column(ForeignKey('c.id')) + b_id = Column(ForeignKey('b.id')) + + # 1. set up the join() as a variable, so we can refer + # to it in the mapping multiple times. + j = join(B, D, D.b_id == B.id).join(C, C.id == D.c_id) + + # 2. Create a new mapper() to B, with non_primary=True. + # Columns in the join with the same name must be + # disambiguated within the mapping, using named properties. + B_viacd = mapper(B, j, non_primary=True, properties={ + "b_id": [j.c.b_id, j.c.d_b_id], + "d_id": j.c.d_id + }) + + A.b = relationship(B_viacd, primaryjoin=A.b_id == B_viacd.c.b_id) + +In the above case, our non-primary mapper for ``B`` will emit for +additional columns when we query; these can be ignored: + +.. sourcecode:: python+sql + + sess.query(A).join(A.b).all() + + {opensql}SELECT a.id AS a_id, a.b_id AS a_b_id + FROM a JOIN (b JOIN d ON d.b_id = b.id JOIN c ON c.id = d.c_id) ON a.b_id = b.id + + +Building Query-Enabled Properties +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Very ambitious custom join conditions may fail to be directly persistable, and +in some cases may not even load correctly. To remove the persistence part of +the equation, use the flag :paramref:`~.relationship.viewonly` on the +:func:`~sqlalchemy.orm.relationship`, which establishes it as a read-only +attribute (data written to the collection will be ignored on flush()). +However, in extreme cases, consider using a regular Python property in +conjunction with :class:`.Query` as follows: + +.. sourcecode:: python+sql + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + + def _get_addresses(self): + return object_session(self).query(Address).with_parent(self).filter(...).all() + addresses = property(_get_addresses) + diff --git a/doc/build/orm/loading.rst b/doc/build/orm/loading.rst deleted file mode 100644 index b2d8124e2..000000000 --- a/doc/build/orm/loading.rst +++ /dev/null @@ -1,546 +0,0 @@ -.. _loading_toplevel: - -.. currentmodule:: sqlalchemy.orm - -Relationship Loading Techniques -=============================== - -A big part of SQLAlchemy is providing a wide range of control over how related objects get loaded when querying. This behavior -can be configured at mapper construction time using the ``lazy`` parameter to the :func:`.relationship` function, -as well as by using options with the :class:`.Query` object. - -Using Loader Strategies: Lazy Loading, Eager Loading ----------------------------------------------------- - -By default, all inter-object relationships are **lazy loading**. The scalar or -collection attribute associated with a :func:`~sqlalchemy.orm.relationship` -contains a trigger which fires the first time the attribute is accessed. This -trigger, in all but one case, issues a SQL call at the point of access -in order to load the related object or objects: - -.. sourcecode:: python+sql - - {sql}>>> jack.addresses - SELECT addresses.id AS addresses_id, addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE ? = addresses.user_id - [5] - {stop}[, ] - -The one case where SQL is not emitted is for a simple many-to-one relationship, when -the related object can be identified by its primary key alone and that object is already -present in the current :class:`.Session`. - -This default behavior of "load upon attribute access" is known as "lazy" or -"select" loading - the name "select" because a "SELECT" statement is typically emitted -when the attribute is first accessed. - -In the :ref:`ormtutorial_toplevel`, we introduced the concept of **Eager -Loading**. We used an ``option`` in conjunction with the -:class:`~sqlalchemy.orm.query.Query` object in order to indicate that a -relationship should be loaded at the same time as the parent, within a single -SQL query. This option, known as :func:`.joinedload`, connects a JOIN (by default -a LEFT OUTER join) to the statement and populates the scalar/collection from the -same result set as that of the parent: - -.. sourcecode:: python+sql - - {sql}>>> jack = session.query(User).\ - ... options(joinedload('addresses')).\ - ... filter_by(name='jack').all() #doctest: +NORMALIZE_WHITESPACE - SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, - addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, - users.fullname AS users_fullname, users.password AS users_password - FROM users LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id - WHERE users.name = ? - ['jack'] - - -In addition to "joined eager loading", a second option for eager loading -exists, called "subquery eager loading". This kind of eager loading emits an -additional SQL statement for each collection requested, aggregated across all -parent objects: - -.. sourcecode:: python+sql - - {sql}>>> jack = session.query(User).\ - ... options(subqueryload('addresses')).\ - ... filter_by(name='jack').all() - SELECT users.id AS users_id, users.name AS users_name, users.fullname AS users_fullname, - users.password AS users_password - FROM users - WHERE users.name = ? - ('jack',) - SELECT addresses.id AS addresses_id, addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id, anon_1.users_id AS anon_1_users_id - FROM (SELECT users.id AS users_id - FROM users - WHERE users.name = ?) AS anon_1 JOIN addresses ON anon_1.users_id = addresses.user_id - ORDER BY anon_1.users_id, addresses.id - ('jack',) - -The default **loader strategy** for any :func:`~sqlalchemy.orm.relationship` -is configured by the ``lazy`` keyword argument, which defaults to ``select`` - this indicates -a "select" statement . -Below we set it as ``joined`` so that the ``children`` relationship is eager -loaded using a JOIN:: - - # load the 'children' collection using LEFT OUTER JOIN - class Parent(Base): - __tablename__ = 'parent' - - id = Column(Integer, primary_key=True) - children = relationship("Child", lazy='joined') - -We can also set it to eagerly load using a second query for all collections, -using ``subquery``:: - - # load the 'children' collection using a second query which - # JOINS to a subquery of the original - class Parent(Base): - __tablename__ = 'parent' - - id = Column(Integer, primary_key=True) - children = relationship("Child", lazy='subquery') - -When querying, all three choices of loader strategy are available on a -per-query basis, using the :func:`~sqlalchemy.orm.joinedload`, -:func:`~sqlalchemy.orm.subqueryload` and :func:`~sqlalchemy.orm.lazyload` -query options: - -.. sourcecode:: python+sql - - # set children to load lazily - session.query(Parent).options(lazyload('children')).all() - - # set children to load eagerly with a join - session.query(Parent).options(joinedload('children')).all() - - # set children to load eagerly with a second statement - session.query(Parent).options(subqueryload('children')).all() - -.. _subqueryload_ordering: - -The Importance of Ordering --------------------------- - -A query which makes use of :func:`.subqueryload` in conjunction with a -limiting modifier such as :meth:`.Query.first`, :meth:`.Query.limit`, -or :meth:`.Query.offset` should **always** include :meth:`.Query.order_by` -against unique column(s) such as the primary key, so that the additional queries -emitted by :func:`.subqueryload` include -the same ordering as used by the parent query. Without it, there is a chance -that the inner query could return the wrong rows:: - - # incorrect, no ORDER BY - session.query(User).options(subqueryload(User.addresses)).first() - - # incorrect if User.name is not unique - session.query(User).options(subqueryload(User.addresses)).order_by(User.name).first() - - # correct - session.query(User).options(subqueryload(User.addresses)).order_by(User.name, User.id).first() - -.. seealso:: - - :ref:`faq_subqueryload_limit_sort` - detailed example - -Loading Along Paths -------------------- - -To reference a relationship that is deeper than one level, method chaining -may be used. The object returned by all loader options is an instance of -the :class:`.Load` class, which provides a so-called "generative" interface:: - - session.query(Parent).options( - joinedload('foo'). - joinedload('bar'). - joinedload('bat') - ).all() - -Using method chaining, the loader style of each link in the path is explicitly -stated. To navigate along a path without changing the existing loader style -of a particular attribute, the :func:`.defaultload` method/function may be used:: - - session.query(A).options( - defaultload("atob").joinedload("btoc") - ).all() - -.. versionchanged:: 0.9.0 - The previous approach of specifying dot-separated paths within loader - options has been superseded by the less ambiguous approach of the - :class:`.Load` object and related methods. With this system, the user - specifies the style of loading for each link along the chain explicitly, - rather than guessing between options like ``joinedload()`` vs. ``joinedload_all()``. - The :func:`.orm.defaultload` is provided to allow path navigation without - modification of existing loader options. The dot-separated path system - as well as the ``_all()`` functions will remain available for backwards- - compatibility indefinitely. - -Default Loading Strategies --------------------------- - -.. versionadded:: 0.7.5 - Default loader strategies as a new feature. - -Each of :func:`.joinedload`, :func:`.subqueryload`, :func:`.lazyload`, -and :func:`.noload` can be used to set the default style of -:func:`.relationship` loading -for a particular query, affecting all :func:`.relationship` -mapped -attributes not otherwise -specified in the :class:`.Query`. This feature is available by passing -the string ``'*'`` as the argument to any of these options:: - - session.query(MyClass).options(lazyload('*')) - -Above, the ``lazyload('*')`` option will supersede the ``lazy`` setting -of all :func:`.relationship` constructs in use for that query, -except for those which use the ``'dynamic'`` style of loading. -If some relationships specify -``lazy='joined'`` or ``lazy='subquery'``, for example, -using ``lazyload('*')`` will unilaterally -cause all those relationships to use ``'select'`` loading, e.g. emit a -SELECT statement when each attribute is accessed. - -The option does not supersede loader options stated in the -query, such as :func:`.eagerload`, -:func:`.subqueryload`, etc. The query below will still use joined loading -for the ``widget`` relationship:: - - session.query(MyClass).options( - lazyload('*'), - joinedload(MyClass.widget) - ) - -If multiple ``'*'`` options are passed, the last one overrides -those previously passed. - -Per-Entity Default Loading Strategies -------------------------------------- - -.. versionadded:: 0.9.0 - Per-entity default loader strategies. - -A variant of the default loader strategy is the ability to set the strategy -on a per-entity basis. For example, if querying for ``User`` and ``Address``, -we can instruct all relationships on ``Address`` only to use lazy loading -by first applying the :class:`.Load` object, then specifying the ``*`` as a -chained option:: - - session.query(User, Address).options(Load(Address).lazyload('*')) - -Above, all relationships on ``Address`` will be set to a lazy load. - -.. _zen_of_eager_loading: - -The Zen of Eager Loading -------------------------- - -The philosophy behind loader strategies is that any set of loading schemes can be -applied to a particular query, and *the results don't change* - only the number -of SQL statements required to fully load related objects and collections changes. A particular -query might start out using all lazy loads. After using it in context, it might be revealed -that particular attributes or collections are always accessed, and that it would be more -efficient to change the loader strategy for these. The strategy can be changed with no other -modifications to the query, the results will remain identical, but fewer SQL statements would be emitted. -In theory (and pretty much in practice), nothing you can do to the :class:`.Query` would make it load -a different set of primary or related objects based on a change in loader strategy. - -How :func:`joinedload` in particular achieves this result of not impacting -entity rows returned in any way is that it creates an anonymous alias of the joins it adds to your -query, so that they can't be referenced by other parts of the query. For example, -the query below uses :func:`.joinedload` to create a LEFT OUTER JOIN from ``users`` -to ``addresses``, however the ``ORDER BY`` added against ``Address.email_address`` -is not valid - the ``Address`` entity is not named in the query: - -.. sourcecode:: python+sql - - >>> jack = session.query(User).\ - ... options(joinedload(User.addresses)).\ - ... filter(User.name=='jack').\ - ... order_by(Address.email_address).all() - {opensql}SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, - addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, - users.fullname AS users_fullname, users.password AS users_password - FROM users LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id - WHERE users.name = ? ORDER BY addresses.email_address <-- this part is wrong ! - ['jack'] - -Above, ``ORDER BY addresses.email_address`` is not valid since ``addresses`` is not in the -FROM list. The correct way to load the ``User`` records and order by email -address is to use :meth:`.Query.join`: - -.. sourcecode:: python+sql - - >>> jack = session.query(User).\ - ... join(User.addresses).\ - ... filter(User.name=='jack').\ - ... order_by(Address.email_address).all() - {opensql} - SELECT users.id AS users_id, users.name AS users_name, - users.fullname AS users_fullname, users.password AS users_password - FROM users JOIN addresses ON users.id = addresses.user_id - WHERE users.name = ? ORDER BY addresses.email_address - ['jack'] - -The statement above is of course not the same as the previous one, in that the columns from ``addresses`` -are not included in the result at all. We can add :func:`.joinedload` back in, so that -there are two joins - one is that which we are ordering on, the other is used anonymously to -load the contents of the ``User.addresses`` collection: - -.. sourcecode:: python+sql - - >>> jack = session.query(User).\ - ... join(User.addresses).\ - ... options(joinedload(User.addresses)).\ - ... filter(User.name=='jack').\ - ... order_by(Address.email_address).all() - {opensql}SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, - addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, - users.fullname AS users_fullname, users.password AS users_password - FROM users JOIN addresses ON users.id = addresses.user_id - LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id - WHERE users.name = ? ORDER BY addresses.email_address - ['jack'] - -What we see above is that our usage of :meth:`.Query.join` is to supply JOIN clauses we'd like -to use in subsequent query criterion, whereas our usage of :func:`.joinedload` only concerns -itself with the loading of the ``User.addresses`` collection, for each ``User`` in the result. -In this case, the two joins most probably appear redundant - which they are. If we -wanted to use just one JOIN for collection loading as well as ordering, we use the -:func:`.contains_eager` option, described in :ref:`contains_eager` below. But -to see why :func:`joinedload` does what it does, consider if we were **filtering** on a -particular ``Address``: - -.. sourcecode:: python+sql - - >>> jack = session.query(User).\ - ... join(User.addresses).\ - ... options(joinedload(User.addresses)).\ - ... filter(User.name=='jack').\ - ... filter(Address.email_address=='someaddress@foo.com').\ - ... all() - {opensql}SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, - addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, - users.fullname AS users_fullname, users.password AS users_password - FROM users JOIN addresses ON users.id = addresses.user_id - LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id - WHERE users.name = ? AND addresses.email_address = ? - ['jack', 'someaddress@foo.com'] - -Above, we can see that the two JOINs have very different roles. One will match exactly -one row, that of the join of ``User`` and ``Address`` where ``Address.email_address=='someaddress@foo.com'``. -The other LEFT OUTER JOIN will match *all* ``Address`` rows related to ``User``, -and is only used to populate the ``User.addresses`` collection, for those ``User`` objects -that are returned. - -By changing the usage of :func:`.joinedload` to another style of loading, we can change -how the collection is loaded completely independently of SQL used to retrieve -the actual ``User`` rows we want. Below we change :func:`.joinedload` into -:func:`.subqueryload`: - -.. sourcecode:: python+sql - - >>> jack = session.query(User).\ - ... join(User.addresses).\ - ... options(subqueryload(User.addresses)).\ - ... filter(User.name=='jack').\ - ... filter(Address.email_address=='someaddress@foo.com').\ - ... all() - {opensql}SELECT users.id AS users_id, users.name AS users_name, - users.fullname AS users_fullname, users.password AS users_password - FROM users JOIN addresses ON users.id = addresses.user_id - WHERE users.name = ? AND addresses.email_address = ? - ['jack', 'someaddress@foo.com'] - - # ... subqueryload() emits a SELECT in order - # to load all address records ... - -When using joined eager loading, if the -query contains a modifier that impacts the rows returned -externally to the joins, such as when using DISTINCT, LIMIT, OFFSET -or equivalent, the completed statement is first -wrapped inside a subquery, and the joins used specifically for joined eager -loading are applied to the subquery. SQLAlchemy's -joined eager loading goes the extra mile, and then ten miles further, to -absolutely ensure that it does not affect the end result of the query, only -the way collections and related objects are loaded, no matter what the format of the query is. - -.. _what_kind_of_loading: - -What Kind of Loading to Use ? ------------------------------ - -Which type of loading to use typically comes down to optimizing the tradeoff -between number of SQL executions, complexity of SQL emitted, and amount of -data fetched. Lets take two examples, a :func:`~sqlalchemy.orm.relationship` -which references a collection, and a :func:`~sqlalchemy.orm.relationship` that -references a scalar many-to-one reference. - -* One to Many Collection - - * When using the default lazy loading, if you load 100 objects, and then access a collection on each of - them, a total of 101 SQL statements will be emitted, although each statement will typically be a - simple SELECT without any joins. - - * When using joined loading, the load of 100 objects and their collections will emit only one SQL - statement. However, the - total number of rows fetched will be equal to the sum of the size of all the collections, plus one - extra row for each parent object that has an empty collection. Each row will also contain the full - set of columns represented by the parents, repeated for each collection item - SQLAlchemy does not - re-fetch these columns other than those of the primary key, however most DBAPIs (with some - exceptions) will transmit the full data of each parent over the wire to the client connection in - any case. Therefore joined eager loading only makes sense when the size of the collections are - relatively small. The LEFT OUTER JOIN can also be performance intensive compared to an INNER join. - - * When using subquery loading, the load of 100 objects will emit two SQL statements. The second - statement will fetch a total number of rows equal to the sum of the size of all collections. An - INNER JOIN is used, and a minimum of parent columns are requested, only the primary keys. So a - subquery load makes sense when the collections are larger. - - * When multiple levels of depth are used with joined or subquery loading, loading collections-within- - collections will multiply the total number of rows fetched in a cartesian fashion. Both forms - of eager loading always join from the original parent class. - -* Many to One Reference - - * When using the default lazy loading, a load of 100 objects will like in the case of the collection - emit as many as 101 SQL statements. However - there is a significant exception to this, in that - if the many-to-one reference is a simple foreign key reference to the target's primary key, each - reference will be checked first in the current identity map using :meth:`.Query.get`. So here, - if the collection of objects references a relatively small set of target objects, or the full set - of possible target objects have already been loaded into the session and are strongly referenced, - using the default of `lazy='select'` is by far the most efficient way to go. - - * When using joined loading, the load of 100 objects will emit only one SQL statement. The join - will be a LEFT OUTER JOIN, and the total number of rows will be equal to 100 in all cases. - If you know that each parent definitely has a child (i.e. the foreign - key reference is NOT NULL), the joined load can be configured with - :paramref:`~.relationship.innerjoin` set to ``True``, which is - usually specified within the :func:`~sqlalchemy.orm.relationship`. For a load of objects where - there are many possible target references which may have not been loaded already, joined loading - with an INNER JOIN is extremely efficient. - - * Subquery loading will issue a second load for all the child objects, so for a load of 100 objects - there would be two SQL statements emitted. There's probably not much advantage here over - joined loading, however, except perhaps that subquery loading can use an INNER JOIN in all cases - whereas joined loading requires that the foreign key is NOT NULL. - -.. _joinedload_and_join: - -.. _contains_eager: - -Routing Explicit Joins/Statements into Eagerly Loaded Collections ------------------------------------------------------------------- - -The behavior of :func:`~sqlalchemy.orm.joinedload()` is such that joins are -created automatically, using anonymous aliases as targets, the results of which -are routed into collections and -scalar references on loaded objects. It is often the case that a query already -includes the necessary joins which represent a particular collection or scalar -reference, and the joins added by the joinedload feature are redundant - yet -you'd still like the collections/references to be populated. - -For this SQLAlchemy supplies the :func:`~sqlalchemy.orm.contains_eager()` -option. This option is used in the same manner as the -:func:`~sqlalchemy.orm.joinedload()` option except it is assumed that the -:class:`~sqlalchemy.orm.query.Query` will specify the appropriate joins -explicitly. Below, we specify a join between ``User`` and ``Address`` -and addtionally establish this as the basis for eager loading of ``User.addresses``:: - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - addresses = relationship("Address") - - class Address(Base): - __tablename__ = 'address' - - # ... - - q = session.query(User).join(User.addresses).\ - options(contains_eager(User.addresses)) - - -If the "eager" portion of the statement is "aliased", the ``alias`` keyword -argument to :func:`~sqlalchemy.orm.contains_eager` may be used to indicate it. -This is sent as a reference to an :func:`.aliased` or :class:`.Alias` -construct: - -.. sourcecode:: python+sql - - # use an alias of the Address entity - adalias = aliased(Address) - - # construct a Query object which expects the "addresses" results - query = session.query(User).\ - outerjoin(adalias, User.addresses).\ - options(contains_eager(User.addresses, alias=adalias)) - - # get results normally - {sql}r = query.all() - SELECT users.user_id AS users_user_id, users.user_name AS users_user_name, adalias.address_id AS adalias_address_id, - adalias.user_id AS adalias_user_id, adalias.email_address AS adalias_email_address, (...other columns...) - FROM users LEFT OUTER JOIN email_addresses AS email_addresses_1 ON users.user_id = email_addresses_1.user_id - -The path given as the argument to :func:`.contains_eager` needs -to be a full path from the starting entity. For example if we were loading -``Users->orders->Order->items->Item``, the string version would look like:: - - query(User).options(contains_eager('orders').contains_eager('items')) - -Or using the class-bound descriptor:: - - query(User).options(contains_eager(User.orders).contains_eager(Order.items)) - -Advanced Usage with Arbitrary Statements -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ``alias`` argument can be more creatively used, in that it can be made -to represent any set of arbitrary names to match up into a statement. -Below it is linked to a :func:`.select` which links a set of column objects -to a string SQL statement:: - - # label the columns of the addresses table - eager_columns = select([ - addresses.c.address_id.label('a1'), - addresses.c.email_address.label('a2'), - addresses.c.user_id.label('a3')]) - - # select from a raw SQL statement which uses those label names for the - # addresses table. contains_eager() matches them up. - query = session.query(User).\ - from_statement("select users.*, addresses.address_id as a1, " - "addresses.email_address as a2, addresses.user_id as a3 " - "from users left outer join addresses on users.user_id=addresses.user_id").\ - options(contains_eager(User.addresses, alias=eager_columns)) - - - -Relationship Loader API ------------------------- - -.. autofunction:: contains_alias - -.. autofunction:: contains_eager - -.. autofunction:: defaultload - -.. autofunction:: eagerload - -.. autofunction:: eagerload_all - -.. autofunction:: immediateload - -.. autofunction:: joinedload - -.. autofunction:: joinedload_all - -.. autofunction:: lazyload - -.. autofunction:: noload - -.. autofunction:: subqueryload - -.. autofunction:: subqueryload_all diff --git a/doc/build/orm/loading_columns.rst b/doc/build/orm/loading_columns.rst new file mode 100644 index 000000000..2d0f02ed5 --- /dev/null +++ b/doc/build/orm/loading_columns.rst @@ -0,0 +1,195 @@ +.. module:: sqlalchemy.orm + +=============== +Loading Columns +=============== + +This section presents additional options regarding the loading of columns. + +.. _deferred: + +Deferred Column Loading +======================== + +This feature allows particular columns of a table be loaded only +upon direct access, instead of when the entity is queried using +:class:`.Query`. This feature is useful when one wants to avoid +loading a large text or binary field into memory when it's not needed. +Individual columns can be lazy loaded by themselves or placed into groups that +lazy-load together, using the :func:`.orm.deferred` function to +mark them as "deferred". In the example below, we define a mapping that will load each of +``.excerpt`` and ``.photo`` in separate, individual-row SELECT statements when each +attribute is first referenced on the individual object instance:: + + from sqlalchemy.orm import deferred + from sqlalchemy import Integer, String, Text, Binary, Column + + class Book(Base): + __tablename__ = 'book' + + book_id = Column(Integer, primary_key=True) + title = Column(String(200), nullable=False) + summary = Column(String(2000)) + excerpt = deferred(Column(Text)) + photo = deferred(Column(Binary)) + +Classical mappings as always place the usage of :func:`.orm.deferred` in the +``properties`` dictionary against the table-bound :class:`.Column`:: + + mapper(Book, book_table, properties={ + 'photo':deferred(book_table.c.photo) + }) + +Deferred columns can be associated with a "group" name, so that they load +together when any of them are first accessed. The example below defines a +mapping with a ``photos`` deferred group. When one ``.photo`` is accessed, all three +photos will be loaded in one SELECT statement. The ``.excerpt`` will be loaded +separately when it is accessed:: + + class Book(Base): + __tablename__ = 'book' + + book_id = Column(Integer, primary_key=True) + title = Column(String(200), nullable=False) + summary = Column(String(2000)) + excerpt = deferred(Column(Text)) + photo1 = deferred(Column(Binary), group='photos') + photo2 = deferred(Column(Binary), group='photos') + photo3 = deferred(Column(Binary), group='photos') + +You can defer or undefer columns at the :class:`~sqlalchemy.orm.query.Query` +level using options, including :func:`.orm.defer` and :func:`.orm.undefer`:: + + from sqlalchemy.orm import defer, undefer + + query = session.query(Book) + query = query.options(defer('summary')) + query = query.options(undefer('excerpt')) + query.all() + +:func:`.orm.deferred` attributes which are marked with a "group" can be undeferred +using :func:`.orm.undefer_group`, sending in the group name:: + + from sqlalchemy.orm import undefer_group + + query = session.query(Book) + query.options(undefer_group('photos')).all() + +Load Only Cols +--------------- + +An arbitrary set of columns can be selected as "load only" columns, which will +be loaded while deferring all other columns on a given entity, using :func:`.orm.load_only`:: + + from sqlalchemy.orm import load_only + + session.query(Book).options(load_only("summary", "excerpt")) + +.. versionadded:: 0.9.0 + +Deferred Loading with Multiple Entities +--------------------------------------- + +To specify column deferral options within a :class:`.Query` that loads multiple types +of entity, the :class:`.Load` object can specify which parent entity to start with:: + + from sqlalchemy.orm import Load + + query = session.query(Book, Author).join(Book.author) + query = query.options( + Load(Book).load_only("summary", "excerpt"), + Load(Author).defer("bio") + ) + +To specify column deferral options along the path of various relationships, +the options support chaining, where the loading style of each relationship +is specified first, then is chained to the deferral options. Such as, to load +``Book`` instances, then joined-eager-load the ``Author``, then apply deferral +options to the ``Author`` entity:: + + from sqlalchemy.orm import joinedload + + query = session.query(Book) + query = query.options( + joinedload(Book.author).load_only("summary", "excerpt"), + ) + +In the case where the loading style of parent relationships should be left +unchanged, use :func:`.orm.defaultload`:: + + from sqlalchemy.orm import defaultload + + query = session.query(Book) + query = query.options( + defaultload(Book.author).load_only("summary", "excerpt"), + ) + +.. versionadded:: 0.9.0 support for :class:`.Load` and other options which + allow for better targeting of deferral options. + +Column Deferral API +------------------- + +.. autofunction:: deferred + +.. autofunction:: defer + +.. autofunction:: load_only + +.. autofunction:: undefer + +.. autofunction:: undefer_group + +.. _bundles: + +Column Bundles +=============== + +The :class:`.Bundle` may be used to query for groups of columns under one +namespace. + +.. versionadded:: 0.9.0 + +The bundle allows columns to be grouped together:: + + from sqlalchemy.orm import Bundle + + bn = Bundle('mybundle', MyClass.data1, MyClass.data2) + for row in session.query(bn).filter(bn.c.data1 == 'd1'): + print row.mybundle.data1, row.mybundle.data2 + +The bundle can be subclassed to provide custom behaviors when results +are fetched. The method :meth:`.Bundle.create_row_processor` is given +the :class:`.Query` and a set of "row processor" functions at query execution +time; these processor functions when given a result row will return the +individual attribute value, which can then be adapted into any kind of +return data structure. Below illustrates replacing the usual :class:`.KeyedTuple` +return structure with a straight Python dictionary:: + + from sqlalchemy.orm import Bundle + + class DictBundle(Bundle): + def create_row_processor(self, query, procs, labels): + """Override create_row_processor to return values as dictionaries""" + def proc(row): + return dict( + zip(labels, (proc(row) for proc in procs)) + ) + return proc + +.. versionchanged:: 1.0 + + The ``proc()`` callable passed to the ``create_row_processor()`` + method of custom :class:`.Bundle` classes now accepts only a single + "row" argument. + +A result from the above bundle will return dictionary values:: + + bn = DictBundle('mybundle', MyClass.data1, MyClass.data2) + for row in session.query(bn).filter(bn.c.data1 == 'd1'): + print row.mybundle['data1'], row.mybundle['data2'] + +The :class:`.Bundle` construct is also integrated into the behavior +of :func:`.composite`, where it is used to return composite attributes as objects +when queried as individual attributes. + diff --git a/doc/build/orm/loading_objects.rst b/doc/build/orm/loading_objects.rst new file mode 100644 index 000000000..e7eb95a3f --- /dev/null +++ b/doc/build/orm/loading_objects.rst @@ -0,0 +1,15 @@ +======================= +Loading Objects +======================= + +Notes and features regarding the general loading of mapped objects. + +For an in-depth introduction to querying with the SQLAlchemy ORM, please see the :ref:`ormtutorial_toplevel`. + +.. toctree:: + :maxdepth: 2 + + loading_columns + loading_relationships + constructors + query diff --git a/doc/build/orm/loading_relationships.rst b/doc/build/orm/loading_relationships.rst new file mode 100644 index 000000000..b2d8124e2 --- /dev/null +++ b/doc/build/orm/loading_relationships.rst @@ -0,0 +1,546 @@ +.. _loading_toplevel: + +.. currentmodule:: sqlalchemy.orm + +Relationship Loading Techniques +=============================== + +A big part of SQLAlchemy is providing a wide range of control over how related objects get loaded when querying. This behavior +can be configured at mapper construction time using the ``lazy`` parameter to the :func:`.relationship` function, +as well as by using options with the :class:`.Query` object. + +Using Loader Strategies: Lazy Loading, Eager Loading +---------------------------------------------------- + +By default, all inter-object relationships are **lazy loading**. The scalar or +collection attribute associated with a :func:`~sqlalchemy.orm.relationship` +contains a trigger which fires the first time the attribute is accessed. This +trigger, in all but one case, issues a SQL call at the point of access +in order to load the related object or objects: + +.. sourcecode:: python+sql + + {sql}>>> jack.addresses + SELECT addresses.id AS addresses_id, addresses.email_address AS addresses_email_address, + addresses.user_id AS addresses_user_id + FROM addresses + WHERE ? = addresses.user_id + [5] + {stop}[, ] + +The one case where SQL is not emitted is for a simple many-to-one relationship, when +the related object can be identified by its primary key alone and that object is already +present in the current :class:`.Session`. + +This default behavior of "load upon attribute access" is known as "lazy" or +"select" loading - the name "select" because a "SELECT" statement is typically emitted +when the attribute is first accessed. + +In the :ref:`ormtutorial_toplevel`, we introduced the concept of **Eager +Loading**. We used an ``option`` in conjunction with the +:class:`~sqlalchemy.orm.query.Query` object in order to indicate that a +relationship should be loaded at the same time as the parent, within a single +SQL query. This option, known as :func:`.joinedload`, connects a JOIN (by default +a LEFT OUTER join) to the statement and populates the scalar/collection from the +same result set as that of the parent: + +.. sourcecode:: python+sql + + {sql}>>> jack = session.query(User).\ + ... options(joinedload('addresses')).\ + ... filter_by(name='jack').all() #doctest: +NORMALIZE_WHITESPACE + SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, + addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, + users.fullname AS users_fullname, users.password AS users_password + FROM users LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id + WHERE users.name = ? + ['jack'] + + +In addition to "joined eager loading", a second option for eager loading +exists, called "subquery eager loading". This kind of eager loading emits an +additional SQL statement for each collection requested, aggregated across all +parent objects: + +.. sourcecode:: python+sql + + {sql}>>> jack = session.query(User).\ + ... options(subqueryload('addresses')).\ + ... filter_by(name='jack').all() + SELECT users.id AS users_id, users.name AS users_name, users.fullname AS users_fullname, + users.password AS users_password + FROM users + WHERE users.name = ? + ('jack',) + SELECT addresses.id AS addresses_id, addresses.email_address AS addresses_email_address, + addresses.user_id AS addresses_user_id, anon_1.users_id AS anon_1_users_id + FROM (SELECT users.id AS users_id + FROM users + WHERE users.name = ?) AS anon_1 JOIN addresses ON anon_1.users_id = addresses.user_id + ORDER BY anon_1.users_id, addresses.id + ('jack',) + +The default **loader strategy** for any :func:`~sqlalchemy.orm.relationship` +is configured by the ``lazy`` keyword argument, which defaults to ``select`` - this indicates +a "select" statement . +Below we set it as ``joined`` so that the ``children`` relationship is eager +loaded using a JOIN:: + + # load the 'children' collection using LEFT OUTER JOIN + class Parent(Base): + __tablename__ = 'parent' + + id = Column(Integer, primary_key=True) + children = relationship("Child", lazy='joined') + +We can also set it to eagerly load using a second query for all collections, +using ``subquery``:: + + # load the 'children' collection using a second query which + # JOINS to a subquery of the original + class Parent(Base): + __tablename__ = 'parent' + + id = Column(Integer, primary_key=True) + children = relationship("Child", lazy='subquery') + +When querying, all three choices of loader strategy are available on a +per-query basis, using the :func:`~sqlalchemy.orm.joinedload`, +:func:`~sqlalchemy.orm.subqueryload` and :func:`~sqlalchemy.orm.lazyload` +query options: + +.. sourcecode:: python+sql + + # set children to load lazily + session.query(Parent).options(lazyload('children')).all() + + # set children to load eagerly with a join + session.query(Parent).options(joinedload('children')).all() + + # set children to load eagerly with a second statement + session.query(Parent).options(subqueryload('children')).all() + +.. _subqueryload_ordering: + +The Importance of Ordering +-------------------------- + +A query which makes use of :func:`.subqueryload` in conjunction with a +limiting modifier such as :meth:`.Query.first`, :meth:`.Query.limit`, +or :meth:`.Query.offset` should **always** include :meth:`.Query.order_by` +against unique column(s) such as the primary key, so that the additional queries +emitted by :func:`.subqueryload` include +the same ordering as used by the parent query. Without it, there is a chance +that the inner query could return the wrong rows:: + + # incorrect, no ORDER BY + session.query(User).options(subqueryload(User.addresses)).first() + + # incorrect if User.name is not unique + session.query(User).options(subqueryload(User.addresses)).order_by(User.name).first() + + # correct + session.query(User).options(subqueryload(User.addresses)).order_by(User.name, User.id).first() + +.. seealso:: + + :ref:`faq_subqueryload_limit_sort` - detailed example + +Loading Along Paths +------------------- + +To reference a relationship that is deeper than one level, method chaining +may be used. The object returned by all loader options is an instance of +the :class:`.Load` class, which provides a so-called "generative" interface:: + + session.query(Parent).options( + joinedload('foo'). + joinedload('bar'). + joinedload('bat') + ).all() + +Using method chaining, the loader style of each link in the path is explicitly +stated. To navigate along a path without changing the existing loader style +of a particular attribute, the :func:`.defaultload` method/function may be used:: + + session.query(A).options( + defaultload("atob").joinedload("btoc") + ).all() + +.. versionchanged:: 0.9.0 + The previous approach of specifying dot-separated paths within loader + options has been superseded by the less ambiguous approach of the + :class:`.Load` object and related methods. With this system, the user + specifies the style of loading for each link along the chain explicitly, + rather than guessing between options like ``joinedload()`` vs. ``joinedload_all()``. + The :func:`.orm.defaultload` is provided to allow path navigation without + modification of existing loader options. The dot-separated path system + as well as the ``_all()`` functions will remain available for backwards- + compatibility indefinitely. + +Default Loading Strategies +-------------------------- + +.. versionadded:: 0.7.5 + Default loader strategies as a new feature. + +Each of :func:`.joinedload`, :func:`.subqueryload`, :func:`.lazyload`, +and :func:`.noload` can be used to set the default style of +:func:`.relationship` loading +for a particular query, affecting all :func:`.relationship` -mapped +attributes not otherwise +specified in the :class:`.Query`. This feature is available by passing +the string ``'*'`` as the argument to any of these options:: + + session.query(MyClass).options(lazyload('*')) + +Above, the ``lazyload('*')`` option will supersede the ``lazy`` setting +of all :func:`.relationship` constructs in use for that query, +except for those which use the ``'dynamic'`` style of loading. +If some relationships specify +``lazy='joined'`` or ``lazy='subquery'``, for example, +using ``lazyload('*')`` will unilaterally +cause all those relationships to use ``'select'`` loading, e.g. emit a +SELECT statement when each attribute is accessed. + +The option does not supersede loader options stated in the +query, such as :func:`.eagerload`, +:func:`.subqueryload`, etc. The query below will still use joined loading +for the ``widget`` relationship:: + + session.query(MyClass).options( + lazyload('*'), + joinedload(MyClass.widget) + ) + +If multiple ``'*'`` options are passed, the last one overrides +those previously passed. + +Per-Entity Default Loading Strategies +------------------------------------- + +.. versionadded:: 0.9.0 + Per-entity default loader strategies. + +A variant of the default loader strategy is the ability to set the strategy +on a per-entity basis. For example, if querying for ``User`` and ``Address``, +we can instruct all relationships on ``Address`` only to use lazy loading +by first applying the :class:`.Load` object, then specifying the ``*`` as a +chained option:: + + session.query(User, Address).options(Load(Address).lazyload('*')) + +Above, all relationships on ``Address`` will be set to a lazy load. + +.. _zen_of_eager_loading: + +The Zen of Eager Loading +------------------------- + +The philosophy behind loader strategies is that any set of loading schemes can be +applied to a particular query, and *the results don't change* - only the number +of SQL statements required to fully load related objects and collections changes. A particular +query might start out using all lazy loads. After using it in context, it might be revealed +that particular attributes or collections are always accessed, and that it would be more +efficient to change the loader strategy for these. The strategy can be changed with no other +modifications to the query, the results will remain identical, but fewer SQL statements would be emitted. +In theory (and pretty much in practice), nothing you can do to the :class:`.Query` would make it load +a different set of primary or related objects based on a change in loader strategy. + +How :func:`joinedload` in particular achieves this result of not impacting +entity rows returned in any way is that it creates an anonymous alias of the joins it adds to your +query, so that they can't be referenced by other parts of the query. For example, +the query below uses :func:`.joinedload` to create a LEFT OUTER JOIN from ``users`` +to ``addresses``, however the ``ORDER BY`` added against ``Address.email_address`` +is not valid - the ``Address`` entity is not named in the query: + +.. sourcecode:: python+sql + + >>> jack = session.query(User).\ + ... options(joinedload(User.addresses)).\ + ... filter(User.name=='jack').\ + ... order_by(Address.email_address).all() + {opensql}SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, + addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, + users.fullname AS users_fullname, users.password AS users_password + FROM users LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id + WHERE users.name = ? ORDER BY addresses.email_address <-- this part is wrong ! + ['jack'] + +Above, ``ORDER BY addresses.email_address`` is not valid since ``addresses`` is not in the +FROM list. The correct way to load the ``User`` records and order by email +address is to use :meth:`.Query.join`: + +.. sourcecode:: python+sql + + >>> jack = session.query(User).\ + ... join(User.addresses).\ + ... filter(User.name=='jack').\ + ... order_by(Address.email_address).all() + {opensql} + SELECT users.id AS users_id, users.name AS users_name, + users.fullname AS users_fullname, users.password AS users_password + FROM users JOIN addresses ON users.id = addresses.user_id + WHERE users.name = ? ORDER BY addresses.email_address + ['jack'] + +The statement above is of course not the same as the previous one, in that the columns from ``addresses`` +are not included in the result at all. We can add :func:`.joinedload` back in, so that +there are two joins - one is that which we are ordering on, the other is used anonymously to +load the contents of the ``User.addresses`` collection: + +.. sourcecode:: python+sql + + >>> jack = session.query(User).\ + ... join(User.addresses).\ + ... options(joinedload(User.addresses)).\ + ... filter(User.name=='jack').\ + ... order_by(Address.email_address).all() + {opensql}SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, + addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, + users.fullname AS users_fullname, users.password AS users_password + FROM users JOIN addresses ON users.id = addresses.user_id + LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id + WHERE users.name = ? ORDER BY addresses.email_address + ['jack'] + +What we see above is that our usage of :meth:`.Query.join` is to supply JOIN clauses we'd like +to use in subsequent query criterion, whereas our usage of :func:`.joinedload` only concerns +itself with the loading of the ``User.addresses`` collection, for each ``User`` in the result. +In this case, the two joins most probably appear redundant - which they are. If we +wanted to use just one JOIN for collection loading as well as ordering, we use the +:func:`.contains_eager` option, described in :ref:`contains_eager` below. But +to see why :func:`joinedload` does what it does, consider if we were **filtering** on a +particular ``Address``: + +.. sourcecode:: python+sql + + >>> jack = session.query(User).\ + ... join(User.addresses).\ + ... options(joinedload(User.addresses)).\ + ... filter(User.name=='jack').\ + ... filter(Address.email_address=='someaddress@foo.com').\ + ... all() + {opensql}SELECT addresses_1.id AS addresses_1_id, addresses_1.email_address AS addresses_1_email_address, + addresses_1.user_id AS addresses_1_user_id, users.id AS users_id, users.name AS users_name, + users.fullname AS users_fullname, users.password AS users_password + FROM users JOIN addresses ON users.id = addresses.user_id + LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id + WHERE users.name = ? AND addresses.email_address = ? + ['jack', 'someaddress@foo.com'] + +Above, we can see that the two JOINs have very different roles. One will match exactly +one row, that of the join of ``User`` and ``Address`` where ``Address.email_address=='someaddress@foo.com'``. +The other LEFT OUTER JOIN will match *all* ``Address`` rows related to ``User``, +and is only used to populate the ``User.addresses`` collection, for those ``User`` objects +that are returned. + +By changing the usage of :func:`.joinedload` to another style of loading, we can change +how the collection is loaded completely independently of SQL used to retrieve +the actual ``User`` rows we want. Below we change :func:`.joinedload` into +:func:`.subqueryload`: + +.. sourcecode:: python+sql + + >>> jack = session.query(User).\ + ... join(User.addresses).\ + ... options(subqueryload(User.addresses)).\ + ... filter(User.name=='jack').\ + ... filter(Address.email_address=='someaddress@foo.com').\ + ... all() + {opensql}SELECT users.id AS users_id, users.name AS users_name, + users.fullname AS users_fullname, users.password AS users_password + FROM users JOIN addresses ON users.id = addresses.user_id + WHERE users.name = ? AND addresses.email_address = ? + ['jack', 'someaddress@foo.com'] + + # ... subqueryload() emits a SELECT in order + # to load all address records ... + +When using joined eager loading, if the +query contains a modifier that impacts the rows returned +externally to the joins, such as when using DISTINCT, LIMIT, OFFSET +or equivalent, the completed statement is first +wrapped inside a subquery, and the joins used specifically for joined eager +loading are applied to the subquery. SQLAlchemy's +joined eager loading goes the extra mile, and then ten miles further, to +absolutely ensure that it does not affect the end result of the query, only +the way collections and related objects are loaded, no matter what the format of the query is. + +.. _what_kind_of_loading: + +What Kind of Loading to Use ? +----------------------------- + +Which type of loading to use typically comes down to optimizing the tradeoff +between number of SQL executions, complexity of SQL emitted, and amount of +data fetched. Lets take two examples, a :func:`~sqlalchemy.orm.relationship` +which references a collection, and a :func:`~sqlalchemy.orm.relationship` that +references a scalar many-to-one reference. + +* One to Many Collection + + * When using the default lazy loading, if you load 100 objects, and then access a collection on each of + them, a total of 101 SQL statements will be emitted, although each statement will typically be a + simple SELECT without any joins. + + * When using joined loading, the load of 100 objects and their collections will emit only one SQL + statement. However, the + total number of rows fetched will be equal to the sum of the size of all the collections, plus one + extra row for each parent object that has an empty collection. Each row will also contain the full + set of columns represented by the parents, repeated for each collection item - SQLAlchemy does not + re-fetch these columns other than those of the primary key, however most DBAPIs (with some + exceptions) will transmit the full data of each parent over the wire to the client connection in + any case. Therefore joined eager loading only makes sense when the size of the collections are + relatively small. The LEFT OUTER JOIN can also be performance intensive compared to an INNER join. + + * When using subquery loading, the load of 100 objects will emit two SQL statements. The second + statement will fetch a total number of rows equal to the sum of the size of all collections. An + INNER JOIN is used, and a minimum of parent columns are requested, only the primary keys. So a + subquery load makes sense when the collections are larger. + + * When multiple levels of depth are used with joined or subquery loading, loading collections-within- + collections will multiply the total number of rows fetched in a cartesian fashion. Both forms + of eager loading always join from the original parent class. + +* Many to One Reference + + * When using the default lazy loading, a load of 100 objects will like in the case of the collection + emit as many as 101 SQL statements. However - there is a significant exception to this, in that + if the many-to-one reference is a simple foreign key reference to the target's primary key, each + reference will be checked first in the current identity map using :meth:`.Query.get`. So here, + if the collection of objects references a relatively small set of target objects, or the full set + of possible target objects have already been loaded into the session and are strongly referenced, + using the default of `lazy='select'` is by far the most efficient way to go. + + * When using joined loading, the load of 100 objects will emit only one SQL statement. The join + will be a LEFT OUTER JOIN, and the total number of rows will be equal to 100 in all cases. + If you know that each parent definitely has a child (i.e. the foreign + key reference is NOT NULL), the joined load can be configured with + :paramref:`~.relationship.innerjoin` set to ``True``, which is + usually specified within the :func:`~sqlalchemy.orm.relationship`. For a load of objects where + there are many possible target references which may have not been loaded already, joined loading + with an INNER JOIN is extremely efficient. + + * Subquery loading will issue a second load for all the child objects, so for a load of 100 objects + there would be two SQL statements emitted. There's probably not much advantage here over + joined loading, however, except perhaps that subquery loading can use an INNER JOIN in all cases + whereas joined loading requires that the foreign key is NOT NULL. + +.. _joinedload_and_join: + +.. _contains_eager: + +Routing Explicit Joins/Statements into Eagerly Loaded Collections +------------------------------------------------------------------ + +The behavior of :func:`~sqlalchemy.orm.joinedload()` is such that joins are +created automatically, using anonymous aliases as targets, the results of which +are routed into collections and +scalar references on loaded objects. It is often the case that a query already +includes the necessary joins which represent a particular collection or scalar +reference, and the joins added by the joinedload feature are redundant - yet +you'd still like the collections/references to be populated. + +For this SQLAlchemy supplies the :func:`~sqlalchemy.orm.contains_eager()` +option. This option is used in the same manner as the +:func:`~sqlalchemy.orm.joinedload()` option except it is assumed that the +:class:`~sqlalchemy.orm.query.Query` will specify the appropriate joins +explicitly. Below, we specify a join between ``User`` and ``Address`` +and addtionally establish this as the basis for eager loading of ``User.addresses``:: + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + addresses = relationship("Address") + + class Address(Base): + __tablename__ = 'address' + + # ... + + q = session.query(User).join(User.addresses).\ + options(contains_eager(User.addresses)) + + +If the "eager" portion of the statement is "aliased", the ``alias`` keyword +argument to :func:`~sqlalchemy.orm.contains_eager` may be used to indicate it. +This is sent as a reference to an :func:`.aliased` or :class:`.Alias` +construct: + +.. sourcecode:: python+sql + + # use an alias of the Address entity + adalias = aliased(Address) + + # construct a Query object which expects the "addresses" results + query = session.query(User).\ + outerjoin(adalias, User.addresses).\ + options(contains_eager(User.addresses, alias=adalias)) + + # get results normally + {sql}r = query.all() + SELECT users.user_id AS users_user_id, users.user_name AS users_user_name, adalias.address_id AS adalias_address_id, + adalias.user_id AS adalias_user_id, adalias.email_address AS adalias_email_address, (...other columns...) + FROM users LEFT OUTER JOIN email_addresses AS email_addresses_1 ON users.user_id = email_addresses_1.user_id + +The path given as the argument to :func:`.contains_eager` needs +to be a full path from the starting entity. For example if we were loading +``Users->orders->Order->items->Item``, the string version would look like:: + + query(User).options(contains_eager('orders').contains_eager('items')) + +Or using the class-bound descriptor:: + + query(User).options(contains_eager(User.orders).contains_eager(Order.items)) + +Advanced Usage with Arbitrary Statements +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``alias`` argument can be more creatively used, in that it can be made +to represent any set of arbitrary names to match up into a statement. +Below it is linked to a :func:`.select` which links a set of column objects +to a string SQL statement:: + + # label the columns of the addresses table + eager_columns = select([ + addresses.c.address_id.label('a1'), + addresses.c.email_address.label('a2'), + addresses.c.user_id.label('a3')]) + + # select from a raw SQL statement which uses those label names for the + # addresses table. contains_eager() matches them up. + query = session.query(User).\ + from_statement("select users.*, addresses.address_id as a1, " + "addresses.email_address as a2, addresses.user_id as a3 " + "from users left outer join addresses on users.user_id=addresses.user_id").\ + options(contains_eager(User.addresses, alias=eager_columns)) + + + +Relationship Loader API +------------------------ + +.. autofunction:: contains_alias + +.. autofunction:: contains_eager + +.. autofunction:: defaultload + +.. autofunction:: eagerload + +.. autofunction:: eagerload_all + +.. autofunction:: immediateload + +.. autofunction:: joinedload + +.. autofunction:: joinedload_all + +.. autofunction:: lazyload + +.. autofunction:: noload + +.. autofunction:: subqueryload + +.. autofunction:: subqueryload_all diff --git a/doc/build/orm/mapped_attributes.rst b/doc/build/orm/mapped_attributes.rst new file mode 100644 index 000000000..2e7e9b3eb --- /dev/null +++ b/doc/build/orm/mapped_attributes.rst @@ -0,0 +1,340 @@ +.. module:: sqlalchemy.orm + +Changing Attribute Behavior +============================ + +.. _simple_validators: + +Simple Validators +----------------- + +A quick way to add a "validation" routine to an attribute is to use the +:func:`~sqlalchemy.orm.validates` decorator. An attribute validator can raise +an exception, halting the process of mutating the attribute's value, or can +change the given value into something different. Validators, like all +attribute extensions, are only called by normal userland code; they are not +issued when the ORM is populating the object:: + + from sqlalchemy.orm import validates + + class EmailAddress(Base): + __tablename__ = 'address' + + id = Column(Integer, primary_key=True) + email = Column(String) + + @validates('email') + def validate_email(self, key, address): + assert '@' in address + return address + +.. versionchanged:: 1.0.0 - validators are no longer triggered within + the flush process when the newly fetched values for primary key + columns as well as some python- or server-side defaults are fetched. + Prior to 1.0, validators may be triggered in those cases as well. + + +Validators also receive collection append events, when items are added to a +collection:: + + from sqlalchemy.orm import validates + + class User(Base): + # ... + + addresses = relationship("Address") + + @validates('addresses') + def validate_address(self, key, address): + assert '@' in address.email + return address + + +The validation function by default does not get emitted for collection +remove events, as the typical expectation is that a value being discarded +doesn't require validation. However, :func:`.validates` supports reception +of these events by specifying ``include_removes=True`` to the decorator. When +this flag is set, the validation function must receive an additional boolean +argument which if ``True`` indicates that the operation is a removal:: + + from sqlalchemy.orm import validates + + class User(Base): + # ... + + addresses = relationship("Address") + + @validates('addresses', include_removes=True) + def validate_address(self, key, address, is_remove): + if is_remove: + raise ValueError( + "not allowed to remove items from the collection") + else: + assert '@' in address.email + return address + +The case where mutually dependent validators are linked via a backref +can also be tailored, using the ``include_backrefs=False`` option; this option, +when set to ``False``, prevents a validation function from emitting if the +event occurs as a result of a backref:: + + from sqlalchemy.orm import validates + + class User(Base): + # ... + + addresses = relationship("Address", backref='user') + + @validates('addresses', include_backrefs=False) + def validate_address(self, key, address): + assert '@' in address.email + return address + +Above, if we were to assign to ``Address.user`` as in ``some_address.user = some_user``, +the ``validate_address()`` function would *not* be emitted, even though an append +occurs to ``some_user.addresses`` - the event is caused by a backref. + +Note that the :func:`~.validates` decorator is a convenience function built on +top of attribute events. An application that requires more control over +configuration of attribute change behavior can make use of this system, +described at :class:`~.AttributeEvents`. + +.. autofunction:: validates + +.. _mapper_hybrids: + +Using Descriptors and Hybrids +----------------------------- + +A more comprehensive way to produce modified behavior for an attribute is to +use :term:`descriptors`. These are commonly used in Python using the ``property()`` +function. The standard SQLAlchemy technique for descriptors is to create a +plain descriptor, and to have it read/write from a mapped attribute with a +different name. Below we illustrate this using Python 2.6-style properties:: + + class EmailAddress(Base): + __tablename__ = 'email_address' + + id = Column(Integer, primary_key=True) + + # name the attribute with an underscore, + # different from the column name + _email = Column("email", String) + + # then create an ".email" attribute + # to get/set "._email" + @property + def email(self): + return self._email + + @email.setter + def email(self, email): + self._email = email + +The approach above will work, but there's more we can add. While our +``EmailAddress`` object will shuttle the value through the ``email`` +descriptor and into the ``_email`` mapped attribute, the class level +``EmailAddress.email`` attribute does not have the usual expression semantics +usable with :class:`.Query`. To provide these, we instead use the +:mod:`~sqlalchemy.ext.hybrid` extension as follows:: + + from sqlalchemy.ext.hybrid import hybrid_property + + class EmailAddress(Base): + __tablename__ = 'email_address' + + id = Column(Integer, primary_key=True) + + _email = Column("email", String) + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, email): + self._email = email + +The ``.email`` attribute, in addition to providing getter/setter behavior when we have an +instance of ``EmailAddress``, also provides a SQL expression when used at the class level, +that is, from the ``EmailAddress`` class directly: + +.. sourcecode:: python+sql + + from sqlalchemy.orm import Session + session = Session() + + {sql}address = session.query(EmailAddress).\ + filter(EmailAddress.email == 'address@example.com').\ + one() + SELECT address.email AS address_email, address.id AS address_id + FROM address + WHERE address.email = ? + ('address@example.com',) + {stop} + + address.email = 'otheraddress@example.com' + {sql}session.commit() + UPDATE address SET email=? WHERE address.id = ? + ('otheraddress@example.com', 1) + COMMIT + {stop} + +The :class:`~.hybrid_property` also allows us to change the behavior of the +attribute, including defining separate behaviors when the attribute is +accessed at the instance level versus at the class/expression level, using the +:meth:`.hybrid_property.expression` modifier. Such as, if we wanted to add a +host name automatically, we might define two sets of string manipulation +logic:: + + class EmailAddress(Base): + __tablename__ = 'email_address' + + id = Column(Integer, primary_key=True) + + _email = Column("email", String) + + @hybrid_property + def email(self): + """Return the value of _email up until the last twelve + characters.""" + + return self._email[:-12] + + @email.setter + def email(self, email): + """Set the value of _email, tacking on the twelve character + value @example.com.""" + + self._email = email + "@example.com" + + @email.expression + def email(cls): + """Produce a SQL expression that represents the value + of the _email column, minus the last twelve characters.""" + + return func.substr(cls._email, 0, func.length(cls._email) - 12) + +Above, accessing the ``email`` property of an instance of ``EmailAddress`` +will return the value of the ``_email`` attribute, removing or adding the +hostname ``@example.com`` from the value. When we query against the ``email`` +attribute, a SQL function is rendered which produces the same effect: + +.. sourcecode:: python+sql + + {sql}address = session.query(EmailAddress).filter(EmailAddress.email == 'address').one() + SELECT address.email AS address_email, address.id AS address_id + FROM address + WHERE substr(address.email, ?, length(address.email) - ?) = ? + (0, 12, 'address') + {stop} + +Read more about Hybrids at :ref:`hybrids_toplevel`. + +.. _synonyms: + +Synonyms +-------- + +Synonyms are a mapper-level construct that allow any attribute on a class +to "mirror" another attribute that is mapped. + +In the most basic sense, the synonym is an easy way to make a certain +attribute available by an additional name:: + + class MyClass(Base): + __tablename__ = 'my_table' + + id = Column(Integer, primary_key=True) + job_status = Column(String(50)) + + status = synonym("job_status") + +The above class ``MyClass`` has two attributes, ``.job_status`` and +``.status`` that will behave as one attribute, both at the expression +level:: + + >>> print MyClass.job_status == 'some_status' + my_table.job_status = :job_status_1 + + >>> print MyClass.status == 'some_status' + my_table.job_status = :job_status_1 + +and at the instance level:: + + >>> m1 = MyClass(status='x') + >>> m1.status, m1.job_status + ('x', 'x') + + >>> m1.job_status = 'y' + >>> m1.status, m1.job_status + ('y', 'y') + +The :func:`.synonym` can be used for any kind of mapped attribute that +subclasses :class:`.MapperProperty`, including mapped columns and relationships, +as well as synonyms themselves. + +Beyond a simple mirror, :func:`.synonym` can also be made to reference +a user-defined :term:`descriptor`. We can supply our +``status`` synonym with a ``@property``:: + + class MyClass(Base): + __tablename__ = 'my_table' + + id = Column(Integer, primary_key=True) + status = Column(String(50)) + + @property + def job_status(self): + return "Status: " + self.status + + job_status = synonym("status", descriptor=job_status) + +When using Declarative, the above pattern can be expressed more succinctly +using the :func:`.synonym_for` decorator:: + + from sqlalchemy.ext.declarative import synonym_for + + class MyClass(Base): + __tablename__ = 'my_table' + + id = Column(Integer, primary_key=True) + status = Column(String(50)) + + @synonym_for("status") + @property + def job_status(self): + return "Status: " + self.status + +While the :func:`.synonym` is useful for simple mirroring, the use case +of augmenting attribute behavior with descriptors is better handled in modern +usage using the :ref:`hybrid attribute ` feature, which +is more oriented towards Python descriptors. Technically, a :func:`.synonym` +can do everything that a :class:`.hybrid_property` can do, as it also supports +injection of custom SQL capabilities, but the hybrid is more straightforward +to use in more complex situations. + +.. autofunction:: synonym + +.. _custom_comparators: + +Operator Customization +---------------------- + +The "operators" used by the SQLAlchemy ORM and Core expression language +are fully customizable. For example, the comparison expression +``User.name == 'ed'`` makes usage of an operator built into Python +itself called ``operator.eq`` - the actual SQL construct which SQLAlchemy +associates with such an operator can be modified. New +operations can be associated with column expressions as well. The operators +which take place for column expressions are most directly redefined at the +type level - see the +section :ref:`types_operators` for a description. + +ORM level functions like :func:`.column_property`, :func:`.relationship`, +and :func:`.composite` also provide for operator redefinition at the ORM +level, by passing a :class:`.PropComparator` subclass to the ``comparator_factory`` +argument of each function. Customization of operators at this level is a +rare use case. See the documentation at :class:`.PropComparator` +for an overview. + diff --git a/doc/build/orm/mapped_sql_expr.rst b/doc/build/orm/mapped_sql_expr.rst new file mode 100644 index 000000000..1ae5b1285 --- /dev/null +++ b/doc/build/orm/mapped_sql_expr.rst @@ -0,0 +1,208 @@ +.. module:: sqlalchemy.orm + +.. _mapper_sql_expressions: + +SQL Expressions as Mapped Attributes +===================================== + +Attributes on a mapped class can be linked to SQL expressions, which can +be used in queries. + +Using a Hybrid +-------------- + +The easiest and most flexible way to link relatively simple SQL expressions to a class is to use a so-called +"hybrid attribute", +described in the section :ref:`hybrids_toplevel`. The hybrid provides +for an expression that works at both the Python level as well as at the +SQL expression level. For example, below we map a class ``User``, +containing attributes ``firstname`` and ``lastname``, and include a hybrid that +will provide for us the ``fullname``, which is the string concatenation of the two:: + + from sqlalchemy.ext.hybrid import hybrid_property + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + firstname = Column(String(50)) + lastname = Column(String(50)) + + @hybrid_property + def fullname(self): + return self.firstname + " " + self.lastname + +Above, the ``fullname`` attribute is interpreted at both the instance and +class level, so that it is available from an instance:: + + some_user = session.query(User).first() + print some_user.fullname + +as well as usable wtihin queries:: + + some_user = session.query(User).filter(User.fullname == "John Smith").first() + +The string concatenation example is a simple one, where the Python expression +can be dual purposed at the instance and class level. Often, the SQL expression +must be distinguished from the Python expression, which can be achieved using +:meth:`.hybrid_property.expression`. Below we illustrate the case where a conditional +needs to be present inside the hybrid, using the ``if`` statement in Python and the +:func:`.sql.expression.case` construct for SQL expressions:: + + from sqlalchemy.ext.hybrid import hybrid_property + from sqlalchemy.sql import case + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + firstname = Column(String(50)) + lastname = Column(String(50)) + + @hybrid_property + def fullname(self): + if self.firstname is not None: + return self.firstname + " " + self.lastname + else: + return self.lastname + + @fullname.expression + def fullname(cls): + return case([ + (cls.firstname != None, cls.firstname + " " + cls.lastname), + ], else_ = cls.lastname) + +.. _mapper_column_property_sql_expressions: + +Using column_property +--------------------- + +The :func:`.orm.column_property` function can be used to map a SQL +expression in a manner similar to a regularly mapped :class:`.Column`. +With this technique, the attribute is loaded +along with all other column-mapped attributes at load time. This is in some +cases an advantage over the usage of hybrids, as the value can be loaded +up front at the same time as the parent row of the object, particularly if +the expression is one which links to other tables (typically as a correlated +subquery) to access data that wouldn't normally be +available on an already loaded object. + +Disadvantages to using :func:`.orm.column_property` for SQL expressions include that +the expression must be compatible with the SELECT statement emitted for the class +as a whole, and there are also some configurational quirks which can occur +when using :func:`.orm.column_property` from declarative mixins. + +Our "fullname" example can be expressed using :func:`.orm.column_property` as +follows:: + + from sqlalchemy.orm import column_property + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + firstname = Column(String(50)) + lastname = Column(String(50)) + fullname = column_property(firstname + " " + lastname) + +Correlated subqueries may be used as well. Below we use the :func:`.select` +construct to create a SELECT that links together the count of ``Address`` +objects available for a particular ``User``:: + + from sqlalchemy.orm import column_property + from sqlalchemy import select, func + from sqlalchemy import Column, Integer, String, ForeignKey + + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + class Address(Base): + __tablename__ = 'address' + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('user.id')) + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + address_count = column_property( + select([func.count(Address.id)]).\ + where(Address.user_id==id).\ + correlate_except(Address) + ) + +In the above example, we define a :func:`.select` construct like the following:: + + select([func.count(Address.id)]).\ + where(Address.user_id==id).\ + correlate_except(Address) + +The meaning of the above statement is, select the count of ``Address.id`` rows +where the ``Address.user_id`` column is equated to ``id``, which in the context +of the ``User`` class is the :class:`.Column` named ``id`` (note that ``id`` is +also the name of a Python built in function, which is not what we want to use +here - if we were outside of the ``User`` class definition, we'd use ``User.id``). + +The :meth:`.select.correlate_except` directive indicates that each element in the +FROM clause of this :func:`.select` may be omitted from the FROM list (that is, correlated +to the enclosing SELECT statement against ``User``) except for the one corresponding +to ``Address``. This isn't strictly necessary, but prevents ``Address`` from +being inadvertently omitted from the FROM list in the case of a long string +of joins between ``User`` and ``Address`` tables where SELECT statements against +``Address`` are nested. + +If import issues prevent the :func:`.column_property` from being defined +inline with the class, it can be assigned to the class after both +are configured. In Declarative this has the effect of calling :meth:`.Mapper.add_property` +to add an additional property after the fact:: + + User.address_count = column_property( + select([func.count(Address.id)]).\ + where(Address.user_id==User.id) + ) + +For many-to-many relationships, use :func:`.and_` to join the fields of the +association table to both tables in a relation, illustrated +here with a classical mapping:: + + from sqlalchemy import and_ + + mapper(Author, authors, properties={ + 'book_count': column_property( + select([func.count(books.c.id)], + and_( + book_authors.c.author_id==authors.c.id, + book_authors.c.book_id==books.c.id + ))) + }) + +Using a plain descriptor +------------------------- + +In cases where a SQL query more elaborate than what :func:`.orm.column_property` +or :class:`.hybrid_property` can provide must be emitted, a regular Python +function accessed as an attribute can be used, assuming the expression +only needs to be available on an already-loaded instance. The function +is decorated with Python's own ``@property`` decorator to mark it as a read-only +attribute. Within the function, :func:`.object_session` +is used to locate the :class:`.Session` corresponding to the current object, +which is then used to emit a query:: + + from sqlalchemy.orm import object_session + from sqlalchemy import select, func + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + firstname = Column(String(50)) + lastname = Column(String(50)) + + @property + def address_count(self): + return object_session(self).\ + scalar( + select([func.count(Address.id)]).\ + where(Address.user_id==self.id) + ) + +The plain descriptor approach is useful as a last resort, but is less performant +in the usual case than both the hybrid and column property approaches, in that +it needs to emit a SQL query upon each access. + diff --git a/doc/build/orm/mapper_config.rst b/doc/build/orm/mapper_config.rst index 8de341a0d..b68c2331d 100644 --- a/doc/build/orm/mapper_config.rst +++ b/doc/build/orm/mapper_config.rst @@ -1,4 +1,3 @@ -.. module:: sqlalchemy.orm .. _mapper_config_toplevel: @@ -10,1663 +9,13 @@ This section describes a variety of configurational patterns that are usable with mappers. It assumes you've worked through :ref:`ormtutorial_toplevel` and know how to construct and use rudimentary mappers and relationships. -.. _classical_mapping: -Classical Mappings -================== - -A *Classical Mapping* refers to the configuration of a mapped class using the -:func:`.mapper` function, without using the Declarative system. As an example, -start with the declarative mapping introduced in :ref:`ormtutorial_toplevel`:: - - class User(Base): - __tablename__ = 'users' - - id = Column(Integer, primary_key=True) - name = Column(String) - fullname = Column(String) - password = Column(String) - -In "classical" form, the table metadata is created separately with the :class:`.Table` -construct, then associated with the ``User`` class via the :func:`.mapper` function:: - - from sqlalchemy import Table, MetaData, Column, ForeignKey, Integer, String - from sqlalchemy.orm import mapper - - metadata = MetaData() - - user = Table('user', metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)), - Column('fullname', String(50)), - Column('password', String(12)) - ) - - class User(object): - def __init__(self, name, fullname, password): - self.name = name - self.fullname = fullname - self.password = password - - mapper(User, user) - -Information about mapped attributes, such as relationships to other classes, are provided -via the ``properties`` dictionary. The example below illustrates a second :class:`.Table` -object, mapped to a class called ``Address``, then linked to ``User`` via :func:`.relationship`:: - - address = Table('address', metadata, - Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey('user.id')), - Column('email_address', String(50)) - ) - - mapper(User, user, properties={ - 'addresses' : relationship(Address, backref='user', order_by=address.c.id) - }) - - mapper(Address, address) - -When using classical mappings, classes must be provided directly without the benefit -of the "string lookup" system provided by Declarative. SQL expressions are typically -specified in terms of the :class:`.Table` objects, i.e. ``address.c.id`` above -for the ``Address`` relationship, and not ``Address.id``, as ``Address`` may not -yet be linked to table metadata, nor can we specify a string here. - -Some examples in the documentation still use the classical approach, but note that -the classical as well as Declarative approaches are **fully interchangeable**. Both -systems ultimately create the same configuration, consisting of a :class:`.Table`, -user-defined class, linked together with a :func:`.mapper`. When we talk about -"the behavior of :func:`.mapper`", this includes when using the Declarative system -as well - it's still used, just behind the scenes. - -Customizing Column Properties -============================== - -The default behavior of :func:`~.orm.mapper` is to assemble all the columns in -the mapped :class:`.Table` into mapped object attributes, each of which are -named according to the name of the column itself (specifically, the ``key`` -attribute of :class:`.Column`). This behavior can be -modified in several ways. - -.. _mapper_column_distinct_names: - -Naming Columns Distinctly from Attribute Names ----------------------------------------------- - -A mapping by default shares the same name for a -:class:`.Column` as that of the mapped attribute - specifically -it matches the :attr:`.Column.key` attribute on :class:`.Column`, which -by default is the same as the :attr:`.Column.name`. - -The name assigned to the Python attribute which maps to -:class:`.Column` can be different from either :attr:`.Column.name` or :attr:`.Column.key` -just by assigning it that way, as we illustrate here in a Declarative mapping:: - - class User(Base): - __tablename__ = 'user' - id = Column('user_id', Integer, primary_key=True) - name = Column('user_name', String(50)) - -Where above ``User.id`` resolves to a column named ``user_id`` -and ``User.name`` resolves to a column named ``user_name``. - -When mapping to an existing table, the :class:`.Column` object -can be referenced directly:: - - class User(Base): - __table__ = user_table - id = user_table.c.user_id - name = user_table.c.user_name - -Or in a classical mapping, placed in the ``properties`` dictionary -with the desired key:: - - mapper(User, user_table, properties={ - 'id': user_table.c.user_id, - 'name': user_table.c.user_name, - }) - -In the next section we'll examine the usage of ``.key`` more closely. - -.. _mapper_automated_reflection_schemes: - -Automating Column Naming Schemes from Reflected Tables ------------------------------------------------------- - -In the previous section :ref:`mapper_column_distinct_names`, we showed how -a :class:`.Column` explicitly mapped to a class can have a different attribute -name than the column. But what if we aren't listing out :class:`.Column` -objects explicitly, and instead are automating the production of :class:`.Table` -objects using reflection (e.g. as described in :ref:`metadata_reflection_toplevel`)? -In this case we can make use of the :meth:`.DDLEvents.column_reflect` event -to intercept the production of :class:`.Column` objects and provide them -with the :attr:`.Column.key` of our choice:: - - @event.listens_for(Table, "column_reflect") - def column_reflect(inspector, table, column_info): - # set column.key = "attr_" - column_info['key'] = "attr_%s" % column_info['name'].lower() - -With the above event, the reflection of :class:`.Column` objects will be intercepted -with our event that adds a new ".key" element, such as in a mapping as below:: - - class MyClass(Base): - __table__ = Table("some_table", Base.metadata, - autoload=True, autoload_with=some_engine) - -If we want to qualify our event to only react for the specific :class:`.MetaData` -object above, we can check for it in our event:: - - @event.listens_for(Table, "column_reflect") - def column_reflect(inspector, table, column_info): - if table.metadata is Base.metadata: - # set column.key = "attr_" - column_info['key'] = "attr_%s" % column_info['name'].lower() - -.. _column_prefix: - -Naming All Columns with a Prefix --------------------------------- - -A quick approach to prefix column names, typically when mapping -to an existing :class:`.Table` object, is to use ``column_prefix``:: - - class User(Base): - __table__ = user_table - __mapper_args__ = {'column_prefix':'_'} - -The above will place attribute names such as ``_user_id``, ``_user_name``, -``_password`` etc. on the mapped ``User`` class. - -This approach is uncommon in modern usage. For dealing with reflected -tables, a more flexible approach is to use that described in -:ref:`mapper_automated_reflection_schemes`. - - -Using column_property for column level options ------------------------------------------------ - -Options can be specified when mapping a :class:`.Column` using the -:func:`.column_property` function. This function -explicitly creates the :class:`.ColumnProperty` used by the -:func:`.mapper` to keep track of the :class:`.Column`; normally, the -:func:`.mapper` creates this automatically. Using :func:`.column_property`, -we can pass additional arguments about how we'd like the :class:`.Column` -to be mapped. Below, we pass an option ``active_history``, -which specifies that a change to this column's value should -result in the former value being loaded first:: - - from sqlalchemy.orm import column_property - - class User(Base): - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - name = column_property(Column(String(50)), active_history=True) - -:func:`.column_property` is also used to map a single attribute to -multiple columns. This use case arises when mapping to a :func:`~.expression.join` -which has attributes which are equated to each other:: - - class User(Base): - __table__ = user.join(address) - - # assign "user.id", "address.user_id" to the - # "id" attribute - id = column_property(user_table.c.id, address_table.c.user_id) - -For more examples featuring this usage, see :ref:`maptojoin`. - -Another place where :func:`.column_property` is needed is to specify SQL expressions as -mapped attributes, such as below where we create an attribute ``fullname`` -that is the string concatenation of the ``firstname`` and ``lastname`` -columns:: - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - firstname = Column(String(50)) - lastname = Column(String(50)) - fullname = column_property(firstname + " " + lastname) - -See examples of this usage at :ref:`mapper_sql_expressions`. - -.. autofunction:: column_property - -.. _include_exclude_cols: - -Mapping a Subset of Table Columns ---------------------------------- - -Sometimes, a :class:`.Table` object was made available using the -reflection process described at :ref:`metadata_reflection` to load -the table's structure from the database. -For such a table that has lots of columns that don't need to be referenced -in the application, the ``include_properties`` or ``exclude_properties`` -arguments can specify that only a subset of columns should be mapped. -For example:: - - class User(Base): - __table__ = user_table - __mapper_args__ = { - 'include_properties' :['user_id', 'user_name'] - } - -...will map the ``User`` class to the ``user_table`` table, only including -the ``user_id`` and ``user_name`` columns - the rest are not referenced. -Similarly:: - - class Address(Base): - __table__ = address_table - __mapper_args__ = { - 'exclude_properties' : ['street', 'city', 'state', 'zip'] - } - -...will map the ``Address`` class to the ``address_table`` table, including -all columns present except ``street``, ``city``, ``state``, and ``zip``. - -When this mapping is used, the columns that are not included will not be -referenced in any SELECT statements emitted by :class:`.Query`, nor will there -be any mapped attribute on the mapped class which represents the column; -assigning an attribute of that name will have no effect beyond that of -a normal Python attribute assignment. - -In some cases, multiple columns may have the same name, such as when -mapping to a join of two or more tables that share some column name. -``include_properties`` and ``exclude_properties`` can also accommodate -:class:`.Column` objects to more accurately describe which columns -should be included or excluded:: - - class UserAddress(Base): - __table__ = user_table.join(addresses_table) - __mapper_args__ = { - 'exclude_properties' :[address_table.c.id], - 'primary_key' : [user_table.c.id] - } - -.. note:: - - insert and update defaults configured on individual - :class:`.Column` objects, i.e. those described at :ref:`metadata_defaults` - including those configured by the ``default``, ``update``, - ``server_default`` and ``server_onupdate`` arguments, will continue to - function normally even if those :class:`.Column` objects are not mapped. - This is because in the case of ``default`` and ``update``, the - :class:`.Column` object is still present on the underlying - :class:`.Table`, thus allowing the default functions to take place when - the ORM emits an INSERT or UPDATE, and in the case of ``server_default`` - and ``server_onupdate``, the relational database itself maintains these - functions. - - -.. _deferred: - -Deferred Column Loading -======================== - -This feature allows particular columns of a table be loaded only -upon direct access, instead of when the entity is queried using -:class:`.Query`. This feature is useful when one wants to avoid -loading a large text or binary field into memory when it's not needed. -Individual columns can be lazy loaded by themselves or placed into groups that -lazy-load together, using the :func:`.orm.deferred` function to -mark them as "deferred". In the example below, we define a mapping that will load each of -``.excerpt`` and ``.photo`` in separate, individual-row SELECT statements when each -attribute is first referenced on the individual object instance:: - - from sqlalchemy.orm import deferred - from sqlalchemy import Integer, String, Text, Binary, Column - - class Book(Base): - __tablename__ = 'book' - - book_id = Column(Integer, primary_key=True) - title = Column(String(200), nullable=False) - summary = Column(String(2000)) - excerpt = deferred(Column(Text)) - photo = deferred(Column(Binary)) - -Classical mappings as always place the usage of :func:`.orm.deferred` in the -``properties`` dictionary against the table-bound :class:`.Column`:: - - mapper(Book, book_table, properties={ - 'photo':deferred(book_table.c.photo) - }) - -Deferred columns can be associated with a "group" name, so that they load -together when any of them are first accessed. The example below defines a -mapping with a ``photos`` deferred group. When one ``.photo`` is accessed, all three -photos will be loaded in one SELECT statement. The ``.excerpt`` will be loaded -separately when it is accessed:: - - class Book(Base): - __tablename__ = 'book' - - book_id = Column(Integer, primary_key=True) - title = Column(String(200), nullable=False) - summary = Column(String(2000)) - excerpt = deferred(Column(Text)) - photo1 = deferred(Column(Binary), group='photos') - photo2 = deferred(Column(Binary), group='photos') - photo3 = deferred(Column(Binary), group='photos') - -You can defer or undefer columns at the :class:`~sqlalchemy.orm.query.Query` -level using options, including :func:`.orm.defer` and :func:`.orm.undefer`:: - - from sqlalchemy.orm import defer, undefer - - query = session.query(Book) - query = query.options(defer('summary')) - query = query.options(undefer('excerpt')) - query.all() - -:func:`.orm.deferred` attributes which are marked with a "group" can be undeferred -using :func:`.orm.undefer_group`, sending in the group name:: - - from sqlalchemy.orm import undefer_group - - query = session.query(Book) - query.options(undefer_group('photos')).all() - -Load Only Cols ---------------- - -An arbitrary set of columns can be selected as "load only" columns, which will -be loaded while deferring all other columns on a given entity, using :func:`.orm.load_only`:: - - from sqlalchemy.orm import load_only - - session.query(Book).options(load_only("summary", "excerpt")) - -.. versionadded:: 0.9.0 - -Deferred Loading with Multiple Entities ---------------------------------------- - -To specify column deferral options within a :class:`.Query` that loads multiple types -of entity, the :class:`.Load` object can specify which parent entity to start with:: - - from sqlalchemy.orm import Load - - query = session.query(Book, Author).join(Book.author) - query = query.options( - Load(Book).load_only("summary", "excerpt"), - Load(Author).defer("bio") - ) - -To specify column deferral options along the path of various relationships, -the options support chaining, where the loading style of each relationship -is specified first, then is chained to the deferral options. Such as, to load -``Book`` instances, then joined-eager-load the ``Author``, then apply deferral -options to the ``Author`` entity:: - - from sqlalchemy.orm import joinedload - - query = session.query(Book) - query = query.options( - joinedload(Book.author).load_only("summary", "excerpt"), - ) - -In the case where the loading style of parent relationships should be left -unchanged, use :func:`.orm.defaultload`:: - - from sqlalchemy.orm import defaultload - - query = session.query(Book) - query = query.options( - defaultload(Book.author).load_only("summary", "excerpt"), - ) - -.. versionadded:: 0.9.0 support for :class:`.Load` and other options which - allow for better targeting of deferral options. - -Column Deferral API -------------------- - -.. autofunction:: deferred - -.. autofunction:: defer - -.. autofunction:: load_only - -.. autofunction:: undefer - -.. autofunction:: undefer_group - -.. _mapper_sql_expressions: - -SQL Expressions as Mapped Attributes -===================================== - -Attributes on a mapped class can be linked to SQL expressions, which can -be used in queries. - -Using a Hybrid --------------- - -The easiest and most flexible way to link relatively simple SQL expressions to a class is to use a so-called -"hybrid attribute", -described in the section :ref:`hybrids_toplevel`. The hybrid provides -for an expression that works at both the Python level as well as at the -SQL expression level. For example, below we map a class ``User``, -containing attributes ``firstname`` and ``lastname``, and include a hybrid that -will provide for us the ``fullname``, which is the string concatenation of the two:: - - from sqlalchemy.ext.hybrid import hybrid_property - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - firstname = Column(String(50)) - lastname = Column(String(50)) - - @hybrid_property - def fullname(self): - return self.firstname + " " + self.lastname - -Above, the ``fullname`` attribute is interpreted at both the instance and -class level, so that it is available from an instance:: - - some_user = session.query(User).first() - print some_user.fullname - -as well as usable wtihin queries:: - - some_user = session.query(User).filter(User.fullname == "John Smith").first() - -The string concatenation example is a simple one, where the Python expression -can be dual purposed at the instance and class level. Often, the SQL expression -must be distinguished from the Python expression, which can be achieved using -:meth:`.hybrid_property.expression`. Below we illustrate the case where a conditional -needs to be present inside the hybrid, using the ``if`` statement in Python and the -:func:`.sql.expression.case` construct for SQL expressions:: - - from sqlalchemy.ext.hybrid import hybrid_property - from sqlalchemy.sql import case - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - firstname = Column(String(50)) - lastname = Column(String(50)) - - @hybrid_property - def fullname(self): - if self.firstname is not None: - return self.firstname + " " + self.lastname - else: - return self.lastname - - @fullname.expression - def fullname(cls): - return case([ - (cls.firstname != None, cls.firstname + " " + cls.lastname), - ], else_ = cls.lastname) - -.. _mapper_column_property_sql_expressions: - -Using column_property ---------------------- - -The :func:`.orm.column_property` function can be used to map a SQL -expression in a manner similar to a regularly mapped :class:`.Column`. -With this technique, the attribute is loaded -along with all other column-mapped attributes at load time. This is in some -cases an advantage over the usage of hybrids, as the value can be loaded -up front at the same time as the parent row of the object, particularly if -the expression is one which links to other tables (typically as a correlated -subquery) to access data that wouldn't normally be -available on an already loaded object. - -Disadvantages to using :func:`.orm.column_property` for SQL expressions include that -the expression must be compatible with the SELECT statement emitted for the class -as a whole, and there are also some configurational quirks which can occur -when using :func:`.orm.column_property` from declarative mixins. - -Our "fullname" example can be expressed using :func:`.orm.column_property` as -follows:: - - from sqlalchemy.orm import column_property - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - firstname = Column(String(50)) - lastname = Column(String(50)) - fullname = column_property(firstname + " " + lastname) - -Correlated subqueries may be used as well. Below we use the :func:`.select` -construct to create a SELECT that links together the count of ``Address`` -objects available for a particular ``User``:: - - from sqlalchemy.orm import column_property - from sqlalchemy import select, func - from sqlalchemy import Column, Integer, String, ForeignKey - - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('user.id')) - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - address_count = column_property( - select([func.count(Address.id)]).\ - where(Address.user_id==id).\ - correlate_except(Address) - ) - -In the above example, we define a :func:`.select` construct like the following:: - - select([func.count(Address.id)]).\ - where(Address.user_id==id).\ - correlate_except(Address) - -The meaning of the above statement is, select the count of ``Address.id`` rows -where the ``Address.user_id`` column is equated to ``id``, which in the context -of the ``User`` class is the :class:`.Column` named ``id`` (note that ``id`` is -also the name of a Python built in function, which is not what we want to use -here - if we were outside of the ``User`` class definition, we'd use ``User.id``). - -The :meth:`.select.correlate_except` directive indicates that each element in the -FROM clause of this :func:`.select` may be omitted from the FROM list (that is, correlated -to the enclosing SELECT statement against ``User``) except for the one corresponding -to ``Address``. This isn't strictly necessary, but prevents ``Address`` from -being inadvertently omitted from the FROM list in the case of a long string -of joins between ``User`` and ``Address`` tables where SELECT statements against -``Address`` are nested. - -If import issues prevent the :func:`.column_property` from being defined -inline with the class, it can be assigned to the class after both -are configured. In Declarative this has the effect of calling :meth:`.Mapper.add_property` -to add an additional property after the fact:: - - User.address_count = column_property( - select([func.count(Address.id)]).\ - where(Address.user_id==User.id) - ) - -For many-to-many relationships, use :func:`.and_` to join the fields of the -association table to both tables in a relation, illustrated -here with a classical mapping:: - - from sqlalchemy import and_ - - mapper(Author, authors, properties={ - 'book_count': column_property( - select([func.count(books.c.id)], - and_( - book_authors.c.author_id==authors.c.id, - book_authors.c.book_id==books.c.id - ))) - }) - -Using a plain descriptor -------------------------- - -In cases where a SQL query more elaborate than what :func:`.orm.column_property` -or :class:`.hybrid_property` can provide must be emitted, a regular Python -function accessed as an attribute can be used, assuming the expression -only needs to be available on an already-loaded instance. The function -is decorated with Python's own ``@property`` decorator to mark it as a read-only -attribute. Within the function, :func:`.object_session` -is used to locate the :class:`.Session` corresponding to the current object, -which is then used to emit a query:: - - from sqlalchemy.orm import object_session - from sqlalchemy import select, func - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - firstname = Column(String(50)) - lastname = Column(String(50)) - - @property - def address_count(self): - return object_session(self).\ - scalar( - select([func.count(Address.id)]).\ - where(Address.user_id==self.id) - ) - -The plain descriptor approach is useful as a last resort, but is less performant -in the usual case than both the hybrid and column property approaches, in that -it needs to emit a SQL query upon each access. - -Changing Attribute Behavior -============================ - -.. _simple_validators: - -Simple Validators ------------------ - -A quick way to add a "validation" routine to an attribute is to use the -:func:`~sqlalchemy.orm.validates` decorator. An attribute validator can raise -an exception, halting the process of mutating the attribute's value, or can -change the given value into something different. Validators, like all -attribute extensions, are only called by normal userland code; they are not -issued when the ORM is populating the object:: - - from sqlalchemy.orm import validates - - class EmailAddress(Base): - __tablename__ = 'address' - - id = Column(Integer, primary_key=True) - email = Column(String) - - @validates('email') - def validate_email(self, key, address): - assert '@' in address - return address - -.. versionchanged:: 1.0.0 - validators are no longer triggered within - the flush process when the newly fetched values for primary key - columns as well as some python- or server-side defaults are fetched. - Prior to 1.0, validators may be triggered in those cases as well. - - -Validators also receive collection append events, when items are added to a -collection:: - - from sqlalchemy.orm import validates - - class User(Base): - # ... - - addresses = relationship("Address") - - @validates('addresses') - def validate_address(self, key, address): - assert '@' in address.email - return address - - -The validation function by default does not get emitted for collection -remove events, as the typical expectation is that a value being discarded -doesn't require validation. However, :func:`.validates` supports reception -of these events by specifying ``include_removes=True`` to the decorator. When -this flag is set, the validation function must receive an additional boolean -argument which if ``True`` indicates that the operation is a removal:: - - from sqlalchemy.orm import validates - - class User(Base): - # ... - - addresses = relationship("Address") - - @validates('addresses', include_removes=True) - def validate_address(self, key, address, is_remove): - if is_remove: - raise ValueError( - "not allowed to remove items from the collection") - else: - assert '@' in address.email - return address - -The case where mutually dependent validators are linked via a backref -can also be tailored, using the ``include_backrefs=False`` option; this option, -when set to ``False``, prevents a validation function from emitting if the -event occurs as a result of a backref:: - - from sqlalchemy.orm import validates - - class User(Base): - # ... - - addresses = relationship("Address", backref='user') - - @validates('addresses', include_backrefs=False) - def validate_address(self, key, address): - assert '@' in address.email - return address - -Above, if we were to assign to ``Address.user`` as in ``some_address.user = some_user``, -the ``validate_address()`` function would *not* be emitted, even though an append -occurs to ``some_user.addresses`` - the event is caused by a backref. - -Note that the :func:`~.validates` decorator is a convenience function built on -top of attribute events. An application that requires more control over -configuration of attribute change behavior can make use of this system, -described at :class:`~.AttributeEvents`. - -.. autofunction:: validates - -.. _mapper_hybrids: - -Using Descriptors and Hybrids ------------------------------ - -A more comprehensive way to produce modified behavior for an attribute is to -use :term:`descriptors`. These are commonly used in Python using the ``property()`` -function. The standard SQLAlchemy technique for descriptors is to create a -plain descriptor, and to have it read/write from a mapped attribute with a -different name. Below we illustrate this using Python 2.6-style properties:: - - class EmailAddress(Base): - __tablename__ = 'email_address' - - id = Column(Integer, primary_key=True) - - # name the attribute with an underscore, - # different from the column name - _email = Column("email", String) - - # then create an ".email" attribute - # to get/set "._email" - @property - def email(self): - return self._email - - @email.setter - def email(self, email): - self._email = email - -The approach above will work, but there's more we can add. While our -``EmailAddress`` object will shuttle the value through the ``email`` -descriptor and into the ``_email`` mapped attribute, the class level -``EmailAddress.email`` attribute does not have the usual expression semantics -usable with :class:`.Query`. To provide these, we instead use the -:mod:`~sqlalchemy.ext.hybrid` extension as follows:: - - from sqlalchemy.ext.hybrid import hybrid_property - - class EmailAddress(Base): - __tablename__ = 'email_address' - - id = Column(Integer, primary_key=True) - - _email = Column("email", String) - - @hybrid_property - def email(self): - return self._email - - @email.setter - def email(self, email): - self._email = email - -The ``.email`` attribute, in addition to providing getter/setter behavior when we have an -instance of ``EmailAddress``, also provides a SQL expression when used at the class level, -that is, from the ``EmailAddress`` class directly: - -.. sourcecode:: python+sql - - from sqlalchemy.orm import Session - session = Session() - - {sql}address = session.query(EmailAddress).\ - filter(EmailAddress.email == 'address@example.com').\ - one() - SELECT address.email AS address_email, address.id AS address_id - FROM address - WHERE address.email = ? - ('address@example.com',) - {stop} - - address.email = 'otheraddress@example.com' - {sql}session.commit() - UPDATE address SET email=? WHERE address.id = ? - ('otheraddress@example.com', 1) - COMMIT - {stop} - -The :class:`~.hybrid_property` also allows us to change the behavior of the -attribute, including defining separate behaviors when the attribute is -accessed at the instance level versus at the class/expression level, using the -:meth:`.hybrid_property.expression` modifier. Such as, if we wanted to add a -host name automatically, we might define two sets of string manipulation -logic:: - - class EmailAddress(Base): - __tablename__ = 'email_address' - - id = Column(Integer, primary_key=True) - - _email = Column("email", String) - - @hybrid_property - def email(self): - """Return the value of _email up until the last twelve - characters.""" - - return self._email[:-12] - - @email.setter - def email(self, email): - """Set the value of _email, tacking on the twelve character - value @example.com.""" - - self._email = email + "@example.com" - - @email.expression - def email(cls): - """Produce a SQL expression that represents the value - of the _email column, minus the last twelve characters.""" - - return func.substr(cls._email, 0, func.length(cls._email) - 12) - -Above, accessing the ``email`` property of an instance of ``EmailAddress`` -will return the value of the ``_email`` attribute, removing or adding the -hostname ``@example.com`` from the value. When we query against the ``email`` -attribute, a SQL function is rendered which produces the same effect: - -.. sourcecode:: python+sql - - {sql}address = session.query(EmailAddress).filter(EmailAddress.email == 'address').one() - SELECT address.email AS address_email, address.id AS address_id - FROM address - WHERE substr(address.email, ?, length(address.email) - ?) = ? - (0, 12, 'address') - {stop} - -Read more about Hybrids at :ref:`hybrids_toplevel`. - -.. _synonyms: - -Synonyms --------- - -Synonyms are a mapper-level construct that allow any attribute on a class -to "mirror" another attribute that is mapped. - -In the most basic sense, the synonym is an easy way to make a certain -attribute available by an additional name:: - - class MyClass(Base): - __tablename__ = 'my_table' - - id = Column(Integer, primary_key=True) - job_status = Column(String(50)) - - status = synonym("job_status") - -The above class ``MyClass`` has two attributes, ``.job_status`` and -``.status`` that will behave as one attribute, both at the expression -level:: - - >>> print MyClass.job_status == 'some_status' - my_table.job_status = :job_status_1 - - >>> print MyClass.status == 'some_status' - my_table.job_status = :job_status_1 - -and at the instance level:: - - >>> m1 = MyClass(status='x') - >>> m1.status, m1.job_status - ('x', 'x') - - >>> m1.job_status = 'y' - >>> m1.status, m1.job_status - ('y', 'y') - -The :func:`.synonym` can be used for any kind of mapped attribute that -subclasses :class:`.MapperProperty`, including mapped columns and relationships, -as well as synonyms themselves. - -Beyond a simple mirror, :func:`.synonym` can also be made to reference -a user-defined :term:`descriptor`. We can supply our -``status`` synonym with a ``@property``:: - - class MyClass(Base): - __tablename__ = 'my_table' - - id = Column(Integer, primary_key=True) - status = Column(String(50)) - - @property - def job_status(self): - return "Status: " + self.status - - job_status = synonym("status", descriptor=job_status) - -When using Declarative, the above pattern can be expressed more succinctly -using the :func:`.synonym_for` decorator:: - - from sqlalchemy.ext.declarative import synonym_for - - class MyClass(Base): - __tablename__ = 'my_table' - - id = Column(Integer, primary_key=True) - status = Column(String(50)) - - @synonym_for("status") - @property - def job_status(self): - return "Status: " + self.status - -While the :func:`.synonym` is useful for simple mirroring, the use case -of augmenting attribute behavior with descriptors is better handled in modern -usage using the :ref:`hybrid attribute ` feature, which -is more oriented towards Python descriptors. Technically, a :func:`.synonym` -can do everything that a :class:`.hybrid_property` can do, as it also supports -injection of custom SQL capabilities, but the hybrid is more straightforward -to use in more complex situations. - -.. autofunction:: synonym - -.. _custom_comparators: - -Operator Customization ----------------------- - -The "operators" used by the SQLAlchemy ORM and Core expression language -are fully customizable. For example, the comparison expression -``User.name == 'ed'`` makes usage of an operator built into Python -itself called ``operator.eq`` - the actual SQL construct which SQLAlchemy -associates with such an operator can be modified. New -operations can be associated with column expressions as well. The operators -which take place for column expressions are most directly redefined at the -type level - see the -section :ref:`types_operators` for a description. - -ORM level functions like :func:`.column_property`, :func:`.relationship`, -and :func:`.composite` also provide for operator redefinition at the ORM -level, by passing a :class:`.PropComparator` subclass to the ``comparator_factory`` -argument of each function. Customization of operators at this level is a -rare use case. See the documentation at :class:`.PropComparator` -for an overview. - -.. _mapper_composite: - -Composite Column Types -======================= - -Sets of columns can be associated with a single user-defined datatype. The ORM -provides a single attribute which represents the group of columns using the -class you provide. - -.. versionchanged:: 0.7 - Composites have been simplified such that - they no longer "conceal" the underlying column based attributes. Additionally, - in-place mutation is no longer automatic; see the section below on - enabling mutability to support tracking of in-place changes. - -.. versionchanged:: 0.9 - Composites will return their object-form, rather than as individual columns, - when used in a column-oriented :class:`.Query` construct. See :ref:`migration_2824`. - -A simple example represents pairs of columns as a ``Point`` object. -``Point`` represents such a pair as ``.x`` and ``.y``:: - - class Point(object): - def __init__(self, x, y): - self.x = x - self.y = y - - def __composite_values__(self): - return self.x, self.y - - def __repr__(self): - return "Point(x=%r, y=%r)" % (self.x, self.y) - - def __eq__(self, other): - return isinstance(other, Point) and \ - other.x == self.x and \ - other.y == self.y - - def __ne__(self, other): - return not self.__eq__(other) - -The requirements for the custom datatype class are that it have a constructor -which accepts positional arguments corresponding to its column format, and -also provides a method ``__composite_values__()`` which returns the state of -the object as a list or tuple, in order of its column-based attributes. It -also should supply adequate ``__eq__()`` and ``__ne__()`` methods which test -the equality of two instances. - -We will create a mapping to a table ``vertice``, which represents two points -as ``x1/y1`` and ``x2/y2``. These are created normally as :class:`.Column` -objects. Then, the :func:`.composite` function is used to assign new -attributes that will represent sets of columns via the ``Point`` class:: - - from sqlalchemy import Column, Integer - from sqlalchemy.orm import composite - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class Vertex(Base): - __tablename__ = 'vertice' - - id = Column(Integer, primary_key=True) - x1 = Column(Integer) - y1 = Column(Integer) - x2 = Column(Integer) - y2 = Column(Integer) - - start = composite(Point, x1, y1) - end = composite(Point, x2, y2) - -A classical mapping above would define each :func:`.composite` -against the existing table:: - - mapper(Vertex, vertice_table, properties={ - 'start':composite(Point, vertice_table.c.x1, vertice_table.c.y1), - 'end':composite(Point, vertice_table.c.x2, vertice_table.c.y2), - }) - -We can now persist and use ``Vertex`` instances, as well as query for them, -using the ``.start`` and ``.end`` attributes against ad-hoc ``Point`` instances: - -.. sourcecode:: python+sql - - >>> v = Vertex(start=Point(3, 4), end=Point(5, 6)) - >>> session.add(v) - >>> q = session.query(Vertex).filter(Vertex.start == Point(3, 4)) - {sql}>>> print q.first().start - BEGIN (implicit) - INSERT INTO vertice (x1, y1, x2, y2) VALUES (?, ?, ?, ?) - (3, 4, 5, 6) - SELECT vertice.id AS vertice_id, - vertice.x1 AS vertice_x1, - vertice.y1 AS vertice_y1, - vertice.x2 AS vertice_x2, - vertice.y2 AS vertice_y2 - FROM vertice - WHERE vertice.x1 = ? AND vertice.y1 = ? - LIMIT ? OFFSET ? - (3, 4, 1, 0) - {stop}Point(x=3, y=4) - -.. autofunction:: composite - - -Tracking In-Place Mutations on Composites ------------------------------------------ - -In-place changes to an existing composite value are -not tracked automatically. Instead, the composite class needs to provide -events to its parent object explicitly. This task is largely automated -via the usage of the :class:`.MutableComposite` mixin, which uses events -to associate each user-defined composite object with all parent associations. -Please see the example in :ref:`mutable_composites`. - -.. versionchanged:: 0.7 - In-place changes to an existing composite value are no longer - tracked automatically; the functionality is superseded by the - :class:`.MutableComposite` class. - -.. _composite_operations: - -Redefining Comparison Operations for Composites ------------------------------------------------ - -The "equals" comparison operation by default produces an AND of all -corresponding columns equated to one another. This can be changed using -the ``comparator_factory`` argument to :func:`.composite`, where we -specify a custom :class:`.CompositeProperty.Comparator` class -to define existing or new operations. -Below we illustrate the "greater than" operator, implementing -the same expression that the base "greater than" does:: - - from sqlalchemy.orm.properties import CompositeProperty - from sqlalchemy import sql - - class PointComparator(CompositeProperty.Comparator): - def __gt__(self, other): - """redefine the 'greater than' operation""" - - return sql.and_(*[a>b for a, b in - zip(self.__clause_element__().clauses, - other.__composite_values__())]) - - class Vertex(Base): - ___tablename__ = 'vertice' - - id = Column(Integer, primary_key=True) - x1 = Column(Integer) - y1 = Column(Integer) - x2 = Column(Integer) - y2 = Column(Integer) - - start = composite(Point, x1, y1, - comparator_factory=PointComparator) - end = composite(Point, x2, y2, - comparator_factory=PointComparator) - -.. _bundles: - -Column Bundles -=============== - -The :class:`.Bundle` may be used to query for groups of columns under one -namespace. - -.. versionadded:: 0.9.0 - -The bundle allows columns to be grouped together:: - - from sqlalchemy.orm import Bundle - - bn = Bundle('mybundle', MyClass.data1, MyClass.data2) - for row in session.query(bn).filter(bn.c.data1 == 'd1'): - print row.mybundle.data1, row.mybundle.data2 - -The bundle can be subclassed to provide custom behaviors when results -are fetched. The method :meth:`.Bundle.create_row_processor` is given -the :class:`.Query` and a set of "row processor" functions at query execution -time; these processor functions when given a result row will return the -individual attribute value, which can then be adapted into any kind of -return data structure. Below illustrates replacing the usual :class:`.KeyedTuple` -return structure with a straight Python dictionary:: - - from sqlalchemy.orm import Bundle - - class DictBundle(Bundle): - def create_row_processor(self, query, procs, labels): - """Override create_row_processor to return values as dictionaries""" - def proc(row): - return dict( - zip(labels, (proc(row) for proc in procs)) - ) - return proc - -.. versionchanged:: 1.0 - - The ``proc()`` callable passed to the ``create_row_processor()`` - method of custom :class:`.Bundle` classes now accepts only a single - "row" argument. - -A result from the above bundle will return dictionary values:: - - bn = DictBundle('mybundle', MyClass.data1, MyClass.data2) - for row in session.query(bn).filter(bn.c.data1 == 'd1'): - print row.mybundle['data1'], row.mybundle['data2'] - -The :class:`.Bundle` construct is also integrated into the behavior -of :func:`.composite`, where it is used to return composite attributes as objects -when queried as individual attributes. - - -.. _maptojoin: - -Mapping a Class against Multiple Tables -======================================== - -Mappers can be constructed against arbitrary relational units (called -*selectables*) in addition to plain tables. For example, the :func:`~.expression.join` -function creates a selectable unit comprised of -multiple tables, complete with its own composite primary key, which can be -mapped in the same way as a :class:`.Table`:: - - from sqlalchemy import Table, Column, Integer, \ - String, MetaData, join, ForeignKey - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import column_property - - metadata = MetaData() - - # define two Table objects - user_table = Table('user', metadata, - Column('id', Integer, primary_key=True), - Column('name', String), - ) - - address_table = Table('address', metadata, - Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey('user.id')), - Column('email_address', String) - ) - - # define a join between them. This - # takes place across the user.id and address.user_id - # columns. - user_address_join = join(user_table, address_table) - - Base = declarative_base() - - # map to it - class AddressUser(Base): - __table__ = user_address_join - - id = column_property(user_table.c.id, address_table.c.user_id) - address_id = address_table.c.id - -In the example above, the join expresses columns for both the -``user`` and the ``address`` table. The ``user.id`` and ``address.user_id`` -columns are equated by foreign key, so in the mapping they are defined -as one attribute, ``AddressUser.id``, using :func:`.column_property` to -indicate a specialized column mapping. Based on this part of the -configuration, the mapping will copy -new primary key values from ``user.id`` into the ``address.user_id`` column -when a flush occurs. - -Additionally, the ``address.id`` column is mapped explicitly to -an attribute named ``address_id``. This is to **disambiguate** the -mapping of the ``address.id`` column from the same-named ``AddressUser.id`` -attribute, which here has been assigned to refer to the ``user`` table -combined with the ``address.user_id`` foreign key. - -The natural primary key of the above mapping is the composite of -``(user.id, address.id)``, as these are the primary key columns of the -``user`` and ``address`` table combined together. The identity of an -``AddressUser`` object will be in terms of these two values, and -is represented from an ``AddressUser`` object as -``(AddressUser.id, AddressUser.address_id)``. - - -Mapping a Class against Arbitrary Selects -========================================= - -Similar to mapping against a join, a plain :func:`~.expression.select` object can be used with a -mapper as well. The example fragment below illustrates mapping a class -called ``Customer`` to a :func:`~.expression.select` which includes a join to a -subquery:: - - from sqlalchemy import select, func - - subq = select([ - func.count(orders.c.id).label('order_count'), - func.max(orders.c.price).label('highest_order'), - orders.c.customer_id - ]).group_by(orders.c.customer_id).alias() - - customer_select = select([customers, subq]).\ - select_from( - join(customers, subq, - customers.c.id == subq.c.customer_id) - ).alias() - - class Customer(Base): - __table__ = customer_select - -Above, the full row represented by ``customer_select`` will be all the -columns of the ``customers`` table, in addition to those columns -exposed by the ``subq`` subquery, which are ``order_count``, -``highest_order``, and ``customer_id``. Mapping the ``Customer`` -class to this selectable then creates a class which will contain -those attributes. - -When the ORM persists new instances of ``Customer``, only the -``customers`` table will actually receive an INSERT. This is because the -primary key of the ``orders`` table is not represented in the mapping; the ORM -will only emit an INSERT into a table for which it has mapped the primary -key. - -.. note:: - - The practice of mapping to arbitrary SELECT statements, especially - complex ones as above, is - almost never needed; it necessarily tends to produce complex queries - which are often less efficient than that which would be produced - by direct query construction. The practice is to some degree - based on the very early history of SQLAlchemy where the :func:`.mapper` - construct was meant to represent the primary querying interface; - in modern usage, the :class:`.Query` object can be used to construct - virtually any SELECT statement, including complex composites, and should - be favored over the "map-to-selectable" approach. - -Multiple Mappers for One Class -============================== - -In modern SQLAlchemy, a particular class is only mapped by one :func:`.mapper` -at a time. The rationale here is that the :func:`.mapper` modifies the class itself, not only -persisting it towards a particular :class:`.Table`, but also *instrumenting* -attributes upon the class which are structured specifically according to the -table metadata. - -One potential use case for another mapper to exist at the same time is if we -wanted to load instances of our class not just from the immediate :class:`.Table` -to which it is mapped, but from another selectable that is a derivation of that -:class:`.Table`. To create a second mapper that only handles querying -when used explicitly, we can use the :paramref:`.mapper.non_primary` argument. -In practice, this approach is usually not needed, as we -can do this sort of thing at query time using methods such as -:meth:`.Query.select_from`, however it is useful in the rare case that we -wish to build a :func:`.relationship` to such a mapper. An example of this is -at :ref:`relationship_non_primary_mapper`. - -Another potential use is if we genuinely want instances of our class to -be persisted into different tables at different times; certain kinds of -data sharding configurations may persist a particular class into tables -that are identical in structure except for their name. For this kind of -pattern, Python offers a better approach than the complexity of mapping -the same class multiple times, which is to instead create new mapped classes -for each target table. SQLAlchemy refers to this as the "entity name" -pattern, which is described as a recipe at `Entity Name -`_. - - -.. _mapping_constructors: - -Constructors and Object Initialization -======================================= - -Mapping imposes no restrictions or requirements on the constructor -(``__init__``) method for the class. You are free to require any arguments for -the function that you wish, assign attributes to the instance that are unknown -to the ORM, and generally do anything else you would normally do when writing -a constructor for a Python class. - -The SQLAlchemy ORM does not call ``__init__`` when recreating objects from -database rows. The ORM's process is somewhat akin to the Python standard -library's ``pickle`` module, invoking the low level ``__new__`` method and -then quietly restoring attributes directly on the instance rather than calling -``__init__``. - -If you need to do some setup on database-loaded instances before they're ready -to use, you can use the ``@reconstructor`` decorator to tag a method as the -ORM counterpart to ``__init__``. SQLAlchemy will call this method with no -arguments every time it loads or reconstructs one of your instances. This is -useful for recreating transient properties that are normally assigned in your -``__init__``:: - - from sqlalchemy import orm - - class MyMappedClass(object): - def __init__(self, data): - self.data = data - # we need stuff on all instances, but not in the database. - self.stuff = [] - - @orm.reconstructor - def init_on_load(self): - self.stuff = [] - -When ``obj = MyMappedClass()`` is executed, Python calls the ``__init__`` -method as normal and the ``data`` argument is required. When instances are -loaded during a :class:`~sqlalchemy.orm.query.Query` operation as in -``query(MyMappedClass).one()``, ``init_on_load`` is called. - -Any method may be tagged as the :func:`~sqlalchemy.orm.reconstructor`, even -the ``__init__`` method. SQLAlchemy will call the reconstructor method with no -arguments. Scalar (non-collection) database-mapped attributes of the instance -will be available for use within the function. Eagerly-loaded collections are -generally not yet available and will usually only contain the first element. -ORM state changes made to objects at this stage will not be recorded for the -next flush() operation, so the activity within a reconstructor should be -conservative. - -:func:`~sqlalchemy.orm.reconstructor` is a shortcut into a larger system -of "instance level" events, which can be subscribed to using the -event API - see :class:`.InstanceEvents` for the full API description -of these events. - -.. autofunction:: reconstructor - - -.. _mapper_version_counter: - -Configuring a Version Counter -============================= - -The :class:`.Mapper` supports management of a :term:`version id column`, which -is a single table column that increments or otherwise updates its value -each time an ``UPDATE`` to the mapped table occurs. This value is checked each -time the ORM emits an ``UPDATE`` or ``DELETE`` against the row to ensure that -the value held in memory matches the database value. - -.. warning:: - - Because the versioning feature relies upon comparison of the **in memory** - record of an object, the feature only applies to the :meth:`.Session.flush` - process, where the ORM flushes individual in-memory rows to the database. - It does **not** take effect when performing - a multirow UPDATE or DELETE using :meth:`.Query.update` or :meth:`.Query.delete` - methods, as these methods only emit an UPDATE or DELETE statement but otherwise - do not have direct access to the contents of those rows being affected. - -The purpose of this feature is to detect when two concurrent transactions -are modifying the same row at roughly the same time, or alternatively to provide -a guard against the usage of a "stale" row in a system that might be re-using -data from a previous transaction without refreshing (e.g. if one sets ``expire_on_commit=False`` -with a :class:`.Session`, it is possible to re-use the data from a previous -transaction). - -.. topic:: Concurrent transaction updates - - When detecting concurrent updates within transactions, it is typically the - case that the database's transaction isolation level is below the level of - :term:`repeatable read`; otherwise, the transaction will not be exposed - to a new row value created by a concurrent update which conflicts with - the locally updated value. In this case, the SQLAlchemy versioning - feature will typically not be useful for in-transaction conflict detection, - though it still can be used for cross-transaction staleness detection. - - The database that enforces repeatable reads will typically either have locked the - target row against a concurrent update, or is employing some form - of multi version concurrency control such that it will emit an error - when the transaction is committed. SQLAlchemy's version_id_col is an alternative - which allows version tracking to occur for specific tables within a transaction - that otherwise might not have this isolation level set. - - .. seealso:: - - `Repeatable Read Isolation Level `_ - Postgresql's implementation of repeatable read, including a description of the error condition. - -Simple Version Counting ------------------------ - -The most straightforward way to track versions is to add an integer column -to the mapped table, then establish it as the ``version_id_col`` within the -mapper options:: - - class User(Base): - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - version_id = Column(Integer, nullable=False) - name = Column(String(50), nullable=False) - - __mapper_args__ = { - "version_id_col": version_id - } - -Above, the ``User`` mapping tracks integer versions using the column -``version_id``. When an object of type ``User`` is first flushed, the -``version_id`` column will be given a value of "1". Then, an UPDATE -of the table later on will always be emitted in a manner similar to the -following:: - - UPDATE user SET version_id=:version_id, name=:name - WHERE user.id = :user_id AND user.version_id = :user_version_id - {"name": "new name", "version_id": 2, "user_id": 1, "user_version_id": 1} - -The above UPDATE statement is updating the row that not only matches -``user.id = 1``, it also is requiring that ``user.version_id = 1``, where "1" -is the last version identifier we've been known to use on this object. -If a transaction elsewhere has modified the row independently, this version id -will no longer match, and the UPDATE statement will report that no rows matched; -this is the condition that SQLAlchemy tests, that exactly one row matched our -UPDATE (or DELETE) statement. If zero rows match, that indicates our version -of the data is stale, and a :exc:`.StaleDataError` is raised. - -.. _custom_version_counter: - -Custom Version Counters / Types -------------------------------- - -Other kinds of values or counters can be used for versioning. Common types include -dates and GUIDs. When using an alternate type or counter scheme, SQLAlchemy -provides a hook for this scheme using the ``version_id_generator`` argument, -which accepts a version generation callable. This callable is passed the value of the current -known version, and is expected to return the subsequent version. - -For example, if we wanted to track the versioning of our ``User`` class -using a randomly generated GUID, we could do this (note that some backends -support a native GUID type, but we illustrate here using a simple string):: - - import uuid - - class User(Base): - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - version_uuid = Column(String(32)) - name = Column(String(50), nullable=False) - - __mapper_args__ = { - 'version_id_col':version_uuid, - 'version_id_generator':lambda version: uuid.uuid4().hex - } - -The persistence engine will call upon ``uuid.uuid4()`` each time a -``User`` object is subject to an INSERT or an UPDATE. In this case, our -version generation function can disregard the incoming value of ``version``, -as the ``uuid4()`` function -generates identifiers without any prerequisite value. If we were using -a sequential versioning scheme such as numeric or a special character system, -we could make use of the given ``version`` in order to help determine the -subsequent value. - -.. seealso:: - - :ref:`custom_guid_type` - -.. _server_side_version_counter: - -Server Side Version Counters ----------------------------- - -The ``version_id_generator`` can also be configured to rely upon a value -that is generated by the database. In this case, the database would need -some means of generating new identifiers when a row is subject to an INSERT -as well as with an UPDATE. For the UPDATE case, typically an update trigger -is needed, unless the database in question supports some other native -version identifier. The Postgresql database in particular supports a system -column called `xmin `_ -which provides UPDATE versioning. We can make use -of the Postgresql ``xmin`` column to version our ``User`` -class as follows:: - - class User(Base): - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - name = Column(String(50), nullable=False) - xmin = Column("xmin", Integer, system=True) - - __mapper_args__ = { - 'version_id_col': xmin, - 'version_id_generator': False - } - -With the above mapping, the ORM will rely upon the ``xmin`` column for -automatically providing the new value of the version id counter. - -.. topic:: creating tables that refer to system columns - - In the above scenario, as ``xmin`` is a system column provided by Postgresql, - we use the ``system=True`` argument to mark it as a system-provided - column, omitted from the ``CREATE TABLE`` statement. - - -The ORM typically does not actively fetch the values of database-generated -values when it emits an INSERT or UPDATE, instead leaving these columns as -"expired" and to be fetched when they are next accessed, unless the ``eager_defaults`` -:func:`.mapper` flag is set. However, when a -server side version column is used, the ORM needs to actively fetch the newly -generated value. This is so that the version counter is set up *before* -any concurrent transaction may update it again. This fetching is also -best done simultaneously within the INSERT or UPDATE statement using :term:`RETURNING`, -otherwise if emitting a SELECT statement afterwards, there is still a potential -race condition where the version counter may change before it can be fetched. - -When the target database supports RETURNING, an INSERT statement for our ``User`` class will look -like this:: - - INSERT INTO "user" (name) VALUES (%(name)s) RETURNING "user".id, "user".xmin - {'name': 'ed'} - -Where above, the ORM can acquire any newly generated primary key values along -with server-generated version identifiers in one statement. When the backend -does not support RETURNING, an additional SELECT must be emitted for **every** -INSERT and UPDATE, which is much less efficient, and also introduces the possibility of -missed version counters:: - - INSERT INTO "user" (name) VALUES (%(name)s) - {'name': 'ed'} - - SELECT "user".version_id AS user_version_id FROM "user" where - "user".id = :param_1 - {"param_1": 1} - -It is *strongly recommended* that server side version counters only be used -when absolutely necessary and only on backends that support :term:`RETURNING`, -e.g. Postgresql, Oracle, SQL Server (though SQL Server has -`major caveats `_ when triggers are used), Firebird. - -.. versionadded:: 0.9.0 - - Support for server side version identifier tracking. - -Programmatic or Conditional Version Counters ---------------------------------------------- - -When ``version_id_generator`` is set to False, we can also programmatically -(and conditionally) set the version identifier on our object in the same way -we assign any other mapped attribute. Such as if we used our UUID example, but -set ``version_id_generator`` to ``False``, we can set the version identifier -at our choosing:: - - import uuid - - class User(Base): - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - version_uuid = Column(String(32)) - name = Column(String(50), nullable=False) - - __mapper_args__ = { - 'version_id_col':version_uuid, - 'version_id_generator': False - } - - u1 = User(name='u1', version_uuid=uuid.uuid4()) - - session.add(u1) - - session.commit() - - u1.name = 'u2' - u1.version_uuid = uuid.uuid4() - - session.commit() - -We can update our ``User`` object without incrementing the version counter -as well; the value of the counter will remain unchanged, and the UPDATE -statement will still check against the previous value. This may be useful -for schemes where only certain classes of UPDATE are sensitive to concurrency -issues:: - - # will leave version_uuid unchanged - u1.name = 'u3' - session.commit() - -.. versionadded:: 0.9.0 - - Support for programmatic and conditional version identifier tracking. - - -Class Mapping API -================= - -.. autofunction:: mapper - -.. autofunction:: object_mapper - -.. autofunction:: class_mapper - -.. autofunction:: configure_mappers - -.. autofunction:: clear_mappers - -.. autofunction:: sqlalchemy.orm.util.identity_key - -.. autofunction:: sqlalchemy.orm.util.polymorphic_union - -.. autoclass:: sqlalchemy.orm.mapper.Mapper - :members: +.. toctree:: + :maxdepth: 2 + classical + scalar_mapping + inheritance + nonstandard_mappings + versioning + mapping_api \ No newline at end of file diff --git a/doc/build/orm/mapping_api.rst b/doc/build/orm/mapping_api.rst new file mode 100644 index 000000000..cd7c379cd --- /dev/null +++ b/doc/build/orm/mapping_api.rst @@ -0,0 +1,22 @@ +.. module:: sqlalchemy.orm + +Class Mapping API +================= + +.. autofunction:: mapper + +.. autofunction:: object_mapper + +.. autofunction:: class_mapper + +.. autofunction:: configure_mappers + +.. autofunction:: clear_mappers + +.. autofunction:: sqlalchemy.orm.util.identity_key + +.. autofunction:: sqlalchemy.orm.util.polymorphic_union + +.. autoclass:: sqlalchemy.orm.mapper.Mapper + :members: + diff --git a/doc/build/orm/mapping_columns.rst b/doc/build/orm/mapping_columns.rst new file mode 100644 index 000000000..b36bfd2f1 --- /dev/null +++ b/doc/build/orm/mapping_columns.rst @@ -0,0 +1,222 @@ +.. module:: sqlalchemy.orm + +Mapping Table Columns +===================== + +The default behavior of :func:`~.orm.mapper` is to assemble all the columns in +the mapped :class:`.Table` into mapped object attributes, each of which are +named according to the name of the column itself (specifically, the ``key`` +attribute of :class:`.Column`). This behavior can be +modified in several ways. + +.. _mapper_column_distinct_names: + +Naming Columns Distinctly from Attribute Names +---------------------------------------------- + +A mapping by default shares the same name for a +:class:`.Column` as that of the mapped attribute - specifically +it matches the :attr:`.Column.key` attribute on :class:`.Column`, which +by default is the same as the :attr:`.Column.name`. + +The name assigned to the Python attribute which maps to +:class:`.Column` can be different from either :attr:`.Column.name` or :attr:`.Column.key` +just by assigning it that way, as we illustrate here in a Declarative mapping:: + + class User(Base): + __tablename__ = 'user' + id = Column('user_id', Integer, primary_key=True) + name = Column('user_name', String(50)) + +Where above ``User.id`` resolves to a column named ``user_id`` +and ``User.name`` resolves to a column named ``user_name``. + +When mapping to an existing table, the :class:`.Column` object +can be referenced directly:: + + class User(Base): + __table__ = user_table + id = user_table.c.user_id + name = user_table.c.user_name + +Or in a classical mapping, placed in the ``properties`` dictionary +with the desired key:: + + mapper(User, user_table, properties={ + 'id': user_table.c.user_id, + 'name': user_table.c.user_name, + }) + +In the next section we'll examine the usage of ``.key`` more closely. + +.. _mapper_automated_reflection_schemes: + +Automating Column Naming Schemes from Reflected Tables +------------------------------------------------------ + +In the previous section :ref:`mapper_column_distinct_names`, we showed how +a :class:`.Column` explicitly mapped to a class can have a different attribute +name than the column. But what if we aren't listing out :class:`.Column` +objects explicitly, and instead are automating the production of :class:`.Table` +objects using reflection (e.g. as described in :ref:`metadata_reflection_toplevel`)? +In this case we can make use of the :meth:`.DDLEvents.column_reflect` event +to intercept the production of :class:`.Column` objects and provide them +with the :attr:`.Column.key` of our choice:: + + @event.listens_for(Table, "column_reflect") + def column_reflect(inspector, table, column_info): + # set column.key = "attr_" + column_info['key'] = "attr_%s" % column_info['name'].lower() + +With the above event, the reflection of :class:`.Column` objects will be intercepted +with our event that adds a new ".key" element, such as in a mapping as below:: + + class MyClass(Base): + __table__ = Table("some_table", Base.metadata, + autoload=True, autoload_with=some_engine) + +If we want to qualify our event to only react for the specific :class:`.MetaData` +object above, we can check for it in our event:: + + @event.listens_for(Table, "column_reflect") + def column_reflect(inspector, table, column_info): + if table.metadata is Base.metadata: + # set column.key = "attr_" + column_info['key'] = "attr_%s" % column_info['name'].lower() + +.. _column_prefix: + +Naming All Columns with a Prefix +-------------------------------- + +A quick approach to prefix column names, typically when mapping +to an existing :class:`.Table` object, is to use ``column_prefix``:: + + class User(Base): + __table__ = user_table + __mapper_args__ = {'column_prefix':'_'} + +The above will place attribute names such as ``_user_id``, ``_user_name``, +``_password`` etc. on the mapped ``User`` class. + +This approach is uncommon in modern usage. For dealing with reflected +tables, a more flexible approach is to use that described in +:ref:`mapper_automated_reflection_schemes`. + + +Using column_property for column level options +----------------------------------------------- + +Options can be specified when mapping a :class:`.Column` using the +:func:`.column_property` function. This function +explicitly creates the :class:`.ColumnProperty` used by the +:func:`.mapper` to keep track of the :class:`.Column`; normally, the +:func:`.mapper` creates this automatically. Using :func:`.column_property`, +we can pass additional arguments about how we'd like the :class:`.Column` +to be mapped. Below, we pass an option ``active_history``, +which specifies that a change to this column's value should +result in the former value being loaded first:: + + from sqlalchemy.orm import column_property + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + name = column_property(Column(String(50)), active_history=True) + +:func:`.column_property` is also used to map a single attribute to +multiple columns. This use case arises when mapping to a :func:`~.expression.join` +which has attributes which are equated to each other:: + + class User(Base): + __table__ = user.join(address) + + # assign "user.id", "address.user_id" to the + # "id" attribute + id = column_property(user_table.c.id, address_table.c.user_id) + +For more examples featuring this usage, see :ref:`maptojoin`. + +Another place where :func:`.column_property` is needed is to specify SQL expressions as +mapped attributes, such as below where we create an attribute ``fullname`` +that is the string concatenation of the ``firstname`` and ``lastname`` +columns:: + + class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + firstname = Column(String(50)) + lastname = Column(String(50)) + fullname = column_property(firstname + " " + lastname) + +See examples of this usage at :ref:`mapper_sql_expressions`. + +.. autofunction:: column_property + +.. _include_exclude_cols: + +Mapping a Subset of Table Columns +--------------------------------- + +Sometimes, a :class:`.Table` object was made available using the +reflection process described at :ref:`metadata_reflection` to load +the table's structure from the database. +For such a table that has lots of columns that don't need to be referenced +in the application, the ``include_properties`` or ``exclude_properties`` +arguments can specify that only a subset of columns should be mapped. +For example:: + + class User(Base): + __table__ = user_table + __mapper_args__ = { + 'include_properties' :['user_id', 'user_name'] + } + +...will map the ``User`` class to the ``user_table`` table, only including +the ``user_id`` and ``user_name`` columns - the rest are not referenced. +Similarly:: + + class Address(Base): + __table__ = address_table + __mapper_args__ = { + 'exclude_properties' : ['street', 'city', 'state', 'zip'] + } + +...will map the ``Address`` class to the ``address_table`` table, including +all columns present except ``street``, ``city``, ``state``, and ``zip``. + +When this mapping is used, the columns that are not included will not be +referenced in any SELECT statements emitted by :class:`.Query`, nor will there +be any mapped attribute on the mapped class which represents the column; +assigning an attribute of that name will have no effect beyond that of +a normal Python attribute assignment. + +In some cases, multiple columns may have the same name, such as when +mapping to a join of two or more tables that share some column name. +``include_properties`` and ``exclude_properties`` can also accommodate +:class:`.Column` objects to more accurately describe which columns +should be included or excluded:: + + class UserAddress(Base): + __table__ = user_table.join(addresses_table) + __mapper_args__ = { + 'exclude_properties' :[address_table.c.id], + 'primary_key' : [user_table.c.id] + } + +.. note:: + + insert and update defaults configured on individual + :class:`.Column` objects, i.e. those described at :ref:`metadata_defaults` + including those configured by the ``default``, ``update``, + ``server_default`` and ``server_onupdate`` arguments, will continue to + function normally even if those :class:`.Column` objects are not mapped. + This is because in the case of ``default`` and ``update``, the + :class:`.Column` object is still present on the underlying + :class:`.Table`, thus allowing the default functions to take place when + the ORM emits an INSERT or UPDATE, and in the case of ``server_default`` + and ``server_onupdate``, the relational database itself maintains these + functions. + + diff --git a/doc/build/orm/nonstandard_mappings.rst b/doc/build/orm/nonstandard_mappings.rst new file mode 100644 index 000000000..b3733a1b9 --- /dev/null +++ b/doc/build/orm/nonstandard_mappings.rst @@ -0,0 +1,152 @@ +======================== +Non-Traditional Mappings +======================== + +.. _maptojoin: + +Mapping a Class against Multiple Tables +======================================== + +Mappers can be constructed against arbitrary relational units (called +*selectables*) in addition to plain tables. For example, the :func:`~.expression.join` +function creates a selectable unit comprised of +multiple tables, complete with its own composite primary key, which can be +mapped in the same way as a :class:`.Table`:: + + from sqlalchemy import Table, Column, Integer, \ + String, MetaData, join, ForeignKey + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import column_property + + metadata = MetaData() + + # define two Table objects + user_table = Table('user', metadata, + Column('id', Integer, primary_key=True), + Column('name', String), + ) + + address_table = Table('address', metadata, + Column('id', Integer, primary_key=True), + Column('user_id', Integer, ForeignKey('user.id')), + Column('email_address', String) + ) + + # define a join between them. This + # takes place across the user.id and address.user_id + # columns. + user_address_join = join(user_table, address_table) + + Base = declarative_base() + + # map to it + class AddressUser(Base): + __table__ = user_address_join + + id = column_property(user_table.c.id, address_table.c.user_id) + address_id = address_table.c.id + +In the example above, the join expresses columns for both the +``user`` and the ``address`` table. The ``user.id`` and ``address.user_id`` +columns are equated by foreign key, so in the mapping they are defined +as one attribute, ``AddressUser.id``, using :func:`.column_property` to +indicate a specialized column mapping. Based on this part of the +configuration, the mapping will copy +new primary key values from ``user.id`` into the ``address.user_id`` column +when a flush occurs. + +Additionally, the ``address.id`` column is mapped explicitly to +an attribute named ``address_id``. This is to **disambiguate** the +mapping of the ``address.id`` column from the same-named ``AddressUser.id`` +attribute, which here has been assigned to refer to the ``user`` table +combined with the ``address.user_id`` foreign key. + +The natural primary key of the above mapping is the composite of +``(user.id, address.id)``, as these are the primary key columns of the +``user`` and ``address`` table combined together. The identity of an +``AddressUser`` object will be in terms of these two values, and +is represented from an ``AddressUser`` object as +``(AddressUser.id, AddressUser.address_id)``. + + +Mapping a Class against Arbitrary Selects +========================================= + +Similar to mapping against a join, a plain :func:`~.expression.select` object can be used with a +mapper as well. The example fragment below illustrates mapping a class +called ``Customer`` to a :func:`~.expression.select` which includes a join to a +subquery:: + + from sqlalchemy import select, func + + subq = select([ + func.count(orders.c.id).label('order_count'), + func.max(orders.c.price).label('highest_order'), + orders.c.customer_id + ]).group_by(orders.c.customer_id).alias() + + customer_select = select([customers, subq]).\ + select_from( + join(customers, subq, + customers.c.id == subq.c.customer_id) + ).alias() + + class Customer(Base): + __table__ = customer_select + +Above, the full row represented by ``customer_select`` will be all the +columns of the ``customers`` table, in addition to those columns +exposed by the ``subq`` subquery, which are ``order_count``, +``highest_order``, and ``customer_id``. Mapping the ``Customer`` +class to this selectable then creates a class which will contain +those attributes. + +When the ORM persists new instances of ``Customer``, only the +``customers`` table will actually receive an INSERT. This is because the +primary key of the ``orders`` table is not represented in the mapping; the ORM +will only emit an INSERT into a table for which it has mapped the primary +key. + +.. note:: + + The practice of mapping to arbitrary SELECT statements, especially + complex ones as above, is + almost never needed; it necessarily tends to produce complex queries + which are often less efficient than that which would be produced + by direct query construction. The practice is to some degree + based on the very early history of SQLAlchemy where the :func:`.mapper` + construct was meant to represent the primary querying interface; + in modern usage, the :class:`.Query` object can be used to construct + virtually any SELECT statement, including complex composites, and should + be favored over the "map-to-selectable" approach. + +Multiple Mappers for One Class +============================== + +In modern SQLAlchemy, a particular class is only mapped by one :func:`.mapper` +at a time. The rationale here is that the :func:`.mapper` modifies the class itself, not only +persisting it towards a particular :class:`.Table`, but also *instrumenting* +attributes upon the class which are structured specifically according to the +table metadata. + +One potential use case for another mapper to exist at the same time is if we +wanted to load instances of our class not just from the immediate :class:`.Table` +to which it is mapped, but from another selectable that is a derivation of that +:class:`.Table`. To create a second mapper that only handles querying +when used explicitly, we can use the :paramref:`.mapper.non_primary` argument. +In practice, this approach is usually not needed, as we +can do this sort of thing at query time using methods such as +:meth:`.Query.select_from`, however it is useful in the rare case that we +wish to build a :func:`.relationship` to such a mapper. An example of this is +at :ref:`relationship_non_primary_mapper`. + +Another potential use is if we genuinely want instances of our class to +be persisted into different tables at different times; certain kinds of +data sharding configurations may persist a particular class into tables +that are identical in structure except for their name. For this kind of +pattern, Python offers a better approach than the complexity of mapping +the same class multiple times, which is to instead create new mapped classes +for each target table. SQLAlchemy refers to this as the "entity name" +pattern, which is described as a recipe at `Entity Name +`_. + diff --git a/doc/build/orm/persistence_techniques.rst b/doc/build/orm/persistence_techniques.rst new file mode 100644 index 000000000..aee48121d --- /dev/null +++ b/doc/build/orm/persistence_techniques.rst @@ -0,0 +1,301 @@ +================================= +Additional Persistence Techniques +================================= + +.. _flush_embedded_sql_expressions: + +Embedding SQL Insert/Update Expressions into a Flush +===================================================== + +This feature allows the value of a database column to be set to a SQL +expression instead of a literal value. It's especially useful for atomic +updates, calling stored procedures, etc. All you do is assign an expression to +an attribute:: + + class SomeClass(object): + pass + mapper(SomeClass, some_table) + + someobject = session.query(SomeClass).get(5) + + # set 'value' attribute to a SQL expression adding one + someobject.value = some_table.c.value + 1 + + # issues "UPDATE some_table SET value=value+1" + session.commit() + +This technique works both for INSERT and UPDATE statements. After the +flush/commit operation, the ``value`` attribute on ``someobject`` above is +expired, so that when next accessed the newly generated value will be loaded +from the database. + +.. _session_sql_expressions: + +Using SQL Expressions with Sessions +==================================== + +SQL expressions and strings can be executed via the +:class:`~sqlalchemy.orm.session.Session` within its transactional context. +This is most easily accomplished using the +:meth:`~.Session.execute` method, which returns a +:class:`~sqlalchemy.engine.ResultProxy` in the same manner as an +:class:`~sqlalchemy.engine.Engine` or +:class:`~sqlalchemy.engine.Connection`:: + + Session = sessionmaker(bind=engine) + session = Session() + + # execute a string statement + result = session.execute("select * from table where id=:id", {'id':7}) + + # execute a SQL expression construct + result = session.execute(select([mytable]).where(mytable.c.id==7)) + +The current :class:`~sqlalchemy.engine.Connection` held by the +:class:`~sqlalchemy.orm.session.Session` is accessible using the +:meth:`~.Session.connection` method:: + + connection = session.connection() + +The examples above deal with a :class:`~sqlalchemy.orm.session.Session` that's +bound to a single :class:`~sqlalchemy.engine.Engine` or +:class:`~sqlalchemy.engine.Connection`. To execute statements using a +:class:`~sqlalchemy.orm.session.Session` which is bound either to multiple +engines, or none at all (i.e. relies upon bound metadata), both +:meth:`~.Session.execute` and +:meth:`~.Session.connection` accept a ``mapper`` keyword +argument, which is passed a mapped class or +:class:`~sqlalchemy.orm.mapper.Mapper` instance, which is used to locate the +proper context for the desired engine:: + + Session = sessionmaker() + session = Session() + + # need to specify mapper or class when executing + result = session.execute("select * from table where id=:id", {'id':7}, mapper=MyMappedClass) + + result = session.execute(select([mytable], mytable.c.id==7), mapper=MyMappedClass) + + connection = session.connection(MyMappedClass) + +.. _session_partitioning: + +Partitioning Strategies +======================= + +Simple Vertical Partitioning +---------------------------- + +Vertical partitioning places different kinds of objects, or different tables, +across multiple databases:: + + engine1 = create_engine('postgresql://db1') + engine2 = create_engine('postgresql://db2') + + Session = sessionmaker(twophase=True) + + # bind User operations to engine 1, Account operations to engine 2 + Session.configure(binds={User:engine1, Account:engine2}) + + session = Session() + +Above, operations against either class will make usage of the :class:`.Engine` +linked to that class. Upon a flush operation, similar rules take place +to ensure each class is written to the right database. + +The transactions among the multiple databases can optionally be coordinated +via two phase commit, if the underlying backend supports it. See +:ref:`session_twophase` for an example. + +Custom Vertical Partitioning +---------------------------- + +More comprehensive rule-based class-level partitioning can be built by +overriding the :meth:`.Session.get_bind` method. Below we illustrate +a custom :class:`.Session` which delivers the following rules: + +1. Flush operations are delivered to the engine named ``master``. + +2. Operations on objects that subclass ``MyOtherClass`` all + occur on the ``other`` engine. + +3. Read operations for all other classes occur on a random + choice of the ``slave1`` or ``slave2`` database. + +:: + + engines = { + 'master':create_engine("sqlite:///master.db"), + 'other':create_engine("sqlite:///other.db"), + 'slave1':create_engine("sqlite:///slave1.db"), + 'slave2':create_engine("sqlite:///slave2.db"), + } + + from sqlalchemy.orm import Session, sessionmaker + import random + + class RoutingSession(Session): + def get_bind(self, mapper=None, clause=None): + if mapper and issubclass(mapper.class_, MyOtherClass): + return engines['other'] + elif self._flushing: + return engines['master'] + else: + return engines[ + random.choice(['slave1','slave2']) + ] + +The above :class:`.Session` class is plugged in using the ``class_`` +argument to :class:`.sessionmaker`:: + + Session = sessionmaker(class_=RoutingSession) + +This approach can be combined with multiple :class:`.MetaData` objects, +using an approach such as that of using the declarative ``__abstract__`` +keyword, described at :ref:`declarative_abstract`. + +Horizontal Partitioning +----------------------- + +Horizontal partitioning partitions the rows of a single table (or a set of +tables) across multiple databases. + +See the "sharding" example: :ref:`examples_sharding`. + +.. _bulk_operations: + +Bulk Operations +=============== + +.. note:: Bulk Operations mode is a new series of operations made available + on the :class:`.Session` object for the purpose of invoking INSERT and + UPDATE statements with greatly reduced Python overhead, at the expense + of much less functionality, automation, and error checking. + As of SQLAlchemy 1.0, these features should be considered as "beta", and + additionally are intended for advanced users. + +.. versionadded:: 1.0.0 + +Bulk operations on the :class:`.Session` include :meth:`.Session.bulk_save_objects`, +:meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings`. +The purpose of these methods is to directly expose internal elements of the unit of work system, +such that facilities for emitting INSERT and UPDATE statements given dictionaries +or object states can be utilized alone, bypassing the normal unit of work +mechanics of state, relationship and attribute management. The advantages +to this approach is strictly one of reduced Python overhead: + +* The flush() process, including the survey of all objects, their state, + their cascade status, the status of all objects associated with them + via :func:`.relationship`, and the topological sort of all operations to + be performed is completely bypassed. This reduces a great amount of + Python overhead. + +* The objects as given have no defined relationship to the target + :class:`.Session`, even when the operation is complete, meaning there's no + overhead in attaching them or managing their state in terms of the identity + map or session. + +* The :meth:`.Session.bulk_insert_mappings` and :meth:`.Session.bulk_update_mappings` + methods accept lists of plain Python dictionaries, not objects; this further + reduces a large amount of overhead associated with instantiating mapped + objects and assigning state to them, which normally is also subject to + expensive tracking of history on a per-attribute basis. + +* The process of fetching primary keys after an INSERT also is disabled by + default. When performed correctly, INSERT statements can now more readily + be batched by the unit of work process into ``executemany()`` blocks, which + perform vastly better than individual statement invocations. + +* UPDATE statements can similarly be tailored such that all attributes + are subject to the SET clase unconditionally, again making it much more + likely that ``executemany()`` blocks can be used. + +The performance behavior of the bulk routines should be studied using the +:ref:`examples_performance` example suite. This is a series of example +scripts which illustrate Python call-counts across a variety of scenarios, +including bulk insert and update scenarios. + +.. seealso:: + + :ref:`examples_performance` - includes detailed examples of bulk operations + contrasted against traditional Core and ORM methods, including performance + metrics. + +Usage +----- + +The methods each work in the context of the :class:`.Session` object's +transaction, like any other:: + + s = Session() + objects = [ + User(name="u1"), + User(name="u2"), + User(name="u3") + ] + s.bulk_save_objects(objects) + +For :meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings`, +dictionaries are passed:: + + s.bulk_insert_mappings(User, + [dict(name="u1"), dict(name="u2"), dict(name="u3")] + ) + +.. seealso:: + + :meth:`.Session.bulk_save_objects` + + :meth:`.Session.bulk_insert_mappings` + + :meth:`.Session.bulk_update_mappings` + + +Comparison to Core Insert / Update Constructs +--------------------------------------------- + +The bulk methods offer performance that under particular circumstances +can be close to that of using the core :class:`.Insert` and +:class:`.Update` constructs in an "executemany" context (for a description +of "executemany", see :ref:`execute_multiple` in the Core tutorial). +In order to achieve this, the +:paramref:`.Session.bulk_insert_mappings.return_defaults` +flag should be disabled so that rows can be batched together. The example +suite in :ref:`examples_performance` should be carefully studied in order +to gain familiarity with how fast bulk performance can be achieved. + +ORM Compatibility +----------------- + +The bulk insert / update methods lose a significant amount of functionality +versus traditional ORM use. The following is a listing of features that +are **not available** when using these methods: + +* persistence along :func:`.relationship` linkages + +* sorting of rows within order of dependency; rows are inserted or updated + directly in the order in which they are passed to the methods + +* Session-management on the given objects, including attachment to the + session, identity map management. + +* Functionality related to primary key mutation, ON UPDATE cascade + +* SQL expression inserts / updates (e.g. :ref:`flush_embedded_sql_expressions`) + +* ORM events such as :meth:`.MapperEvents.before_insert`, etc. The bulk + session methods have no event support. + +Features that **are available** include: + +* INSERTs and UPDATEs of mapped objects + +* Version identifier support + +* Multi-table mappings, such as joined-inheritance - however, an object + to be inserted across multiple tables either needs to have primary key + identifiers fully populated ahead of time, else the + :paramref:`.Session.bulk_save_objects.return_defaults` flag must be used, + which will greatly reduce the performance benefits + + diff --git a/doc/build/orm/query.rst b/doc/build/orm/query.rst index 5e31d710f..1517cb997 100644 --- a/doc/build/orm/query.rst +++ b/doc/build/orm/query.rst @@ -1,15 +1,9 @@ .. _query_api_toplevel: - -Querying -======== - -This section provides API documentation for the :class:`.Query` object and related constructs. - -For an in-depth introduction to querying with the SQLAlchemy ORM, please see the :ref:`ormtutorial_toplevel`. - - .. module:: sqlalchemy.orm +Query API +========= + The Query Object ---------------- diff --git a/doc/build/orm/relationship_api.rst b/doc/build/orm/relationship_api.rst new file mode 100644 index 000000000..03045f698 --- /dev/null +++ b/doc/build/orm/relationship_api.rst @@ -0,0 +1,19 @@ +.. automodule:: sqlalchemy.orm + +Relationships API +----------------- + +.. autofunction:: relationship + +.. autofunction:: backref + +.. autofunction:: relation + +.. autofunction:: dynamic_loader + +.. autofunction:: foreign + +.. autofunction:: remote + + + diff --git a/doc/build/orm/relationship_persistence.rst b/doc/build/orm/relationship_persistence.rst new file mode 100644 index 000000000..6d2ba7882 --- /dev/null +++ b/doc/build/orm/relationship_persistence.rst @@ -0,0 +1,229 @@ +Special Relationship Persistence Patterns +========================================= + +.. _post_update: + +Rows that point to themselves / Mutually Dependent Rows +------------------------------------------------------- + +This is a very specific case where relationship() must perform an INSERT and a +second UPDATE in order to properly populate a row (and vice versa an UPDATE +and DELETE in order to delete without violating foreign key constraints). The +two use cases are: + +* A table contains a foreign key to itself, and a single row will + have a foreign key value pointing to its own primary key. +* Two tables each contain a foreign key referencing the other + table, with a row in each table referencing the other. + +For example:: + + user + --------------------------------- + user_id name related_user_id + 1 'ed' 1 + +Or:: + + widget entry + ------------------------------------------- --------------------------------- + widget_id name favorite_entry_id entry_id name widget_id + 1 'somewidget' 5 5 'someentry' 1 + +In the first case, a row points to itself. Technically, a database that uses +sequences such as PostgreSQL or Oracle can INSERT the row at once using a +previously generated value, but databases which rely upon autoincrement-style +primary key identifiers cannot. The :func:`~sqlalchemy.orm.relationship` +always assumes a "parent/child" model of row population during flush, so +unless you are populating the primary key/foreign key columns directly, +:func:`~sqlalchemy.orm.relationship` needs to use two statements. + +In the second case, the "widget" row must be inserted before any referring +"entry" rows, but then the "favorite_entry_id" column of that "widget" row +cannot be set until the "entry" rows have been generated. In this case, it's +typically impossible to insert the "widget" and "entry" rows using just two +INSERT statements; an UPDATE must be performed in order to keep foreign key +constraints fulfilled. The exception is if the foreign keys are configured as +"deferred until commit" (a feature some databases support) and if the +identifiers were populated manually (again essentially bypassing +:func:`~sqlalchemy.orm.relationship`). + +To enable the usage of a supplementary UPDATE statement, +we use the :paramref:`~.relationship.post_update` option +of :func:`.relationship`. This specifies that the linkage between the +two rows should be created using an UPDATE statement after both rows +have been INSERTED; it also causes the rows to be de-associated with +each other via UPDATE before a DELETE is emitted. The flag should +be placed on just *one* of the relationships, preferably the +many-to-one side. Below we illustrate +a complete example, including two :class:`.ForeignKey` constructs, one which +specifies :paramref:`~.ForeignKey.use_alter` to help with emitting CREATE TABLE statements:: + + from sqlalchemy import Integer, ForeignKey, Column + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class Entry(Base): + __tablename__ = 'entry' + entry_id = Column(Integer, primary_key=True) + widget_id = Column(Integer, ForeignKey('widget.widget_id')) + name = Column(String(50)) + + class Widget(Base): + __tablename__ = 'widget' + + widget_id = Column(Integer, primary_key=True) + favorite_entry_id = Column(Integer, + ForeignKey('entry.entry_id', + use_alter=True, + name="fk_favorite_entry")) + name = Column(String(50)) + + entries = relationship(Entry, primaryjoin= + widget_id==Entry.widget_id) + favorite_entry = relationship(Entry, + primaryjoin= + favorite_entry_id==Entry.entry_id, + post_update=True) + +When a structure against the above configuration is flushed, the "widget" row will be +INSERTed minus the "favorite_entry_id" value, then all the "entry" rows will +be INSERTed referencing the parent "widget" row, and then an UPDATE statement +will populate the "favorite_entry_id" column of the "widget" table (it's one +row at a time for the time being): + +.. sourcecode:: pycon+sql + + >>> w1 = Widget(name='somewidget') + >>> e1 = Entry(name='someentry') + >>> w1.favorite_entry = e1 + >>> w1.entries = [e1] + >>> session.add_all([w1, e1]) + {sql}>>> session.commit() + BEGIN (implicit) + INSERT INTO widget (favorite_entry_id, name) VALUES (?, ?) + (None, 'somewidget') + INSERT INTO entry (widget_id, name) VALUES (?, ?) + (1, 'someentry') + UPDATE widget SET favorite_entry_id=? WHERE widget.widget_id = ? + (1, 1) + COMMIT + +An additional configuration we can specify is to supply a more +comprehensive foreign key constraint on ``Widget``, such that +it's guaranteed that ``favorite_entry_id`` refers to an ``Entry`` +that also refers to this ``Widget``. We can use a composite foreign key, +as illustrated below:: + + from sqlalchemy import Integer, ForeignKey, String, \ + Column, UniqueConstraint, ForeignKeyConstraint + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import relationship + + Base = declarative_base() + + class Entry(Base): + __tablename__ = 'entry' + entry_id = Column(Integer, primary_key=True) + widget_id = Column(Integer, ForeignKey('widget.widget_id')) + name = Column(String(50)) + __table_args__ = ( + UniqueConstraint("entry_id", "widget_id"), + ) + + class Widget(Base): + __tablename__ = 'widget' + + widget_id = Column(Integer, autoincrement='ignore_fk', primary_key=True) + favorite_entry_id = Column(Integer) + + name = Column(String(50)) + + __table_args__ = ( + ForeignKeyConstraint( + ["widget_id", "favorite_entry_id"], + ["entry.widget_id", "entry.entry_id"], + name="fk_favorite_entry", use_alter=True + ), + ) + + entries = relationship(Entry, primaryjoin= + widget_id==Entry.widget_id, + foreign_keys=Entry.widget_id) + favorite_entry = relationship(Entry, + primaryjoin= + favorite_entry_id==Entry.entry_id, + foreign_keys=favorite_entry_id, + post_update=True) + +The above mapping features a composite :class:`.ForeignKeyConstraint` +bridging the ``widget_id`` and ``favorite_entry_id`` columns. To ensure +that ``Widget.widget_id`` remains an "autoincrementing" column we specify +:paramref:`~.Column.autoincrement` to the value ``"ignore_fk"`` +on :class:`.Column`, and additionally on each +:func:`.relationship` we must limit those columns considered as part of +the foreign key for the purposes of joining and cross-population. + +.. _passive_updates: + +Mutable Primary Keys / Update Cascades +--------------------------------------- + +When the primary key of an entity changes, related items +which reference the primary key must also be updated as +well. For databases which enforce referential integrity, +it's required to use the database's ON UPDATE CASCADE +functionality in order to propagate primary key changes +to referenced foreign keys - the values cannot be out +of sync for any moment. + +For databases that don't support this, such as SQLite and +MySQL without their referential integrity options turned +on, the :paramref:`~.relationship.passive_updates` flag can +be set to ``False``, most preferably on a one-to-many or +many-to-many :func:`.relationship`, which instructs +SQLAlchemy to issue UPDATE statements individually for +objects referenced in the collection, loading them into +memory if not already locally present. The +:paramref:`~.relationship.passive_updates` flag can also be ``False`` in +conjunction with ON UPDATE CASCADE functionality, +although in that case the unit of work will be issuing +extra SELECT and UPDATE statements unnecessarily. + +A typical mutable primary key setup might look like:: + + class User(Base): + __tablename__ = 'user' + + username = Column(String(50), primary_key=True) + fullname = Column(String(100)) + + # passive_updates=False *only* needed if the database + # does not implement ON UPDATE CASCADE + addresses = relationship("Address", passive_updates=False) + + class Address(Base): + __tablename__ = 'address' + + email = Column(String(50), primary_key=True) + username = Column(String(50), + ForeignKey('user.username', onupdate="cascade") + ) + +:paramref:`~.relationship.passive_updates` is set to ``True`` by default, +indicating that ON UPDATE CASCADE is expected to be in +place in the usual case for foreign keys that expect +to have a mutating parent key. + +A :paramref:`~.relationship.passive_updates` setting of False may be configured on any +direction of relationship, i.e. one-to-many, many-to-one, +and many-to-many, although it is much more effective when +placed just on the one-to-many or many-to-many side. +Configuring the :paramref:`~.relationship.passive_updates` +to False only on the +many-to-one side will have only a partial effect, as the +unit of work searches only through the current identity +map for objects that may be referencing the one with a +mutating primary key, not throughout the database. diff --git a/doc/build/orm/relationships.rst b/doc/build/orm/relationships.rst index f512251a7..6fea107a7 100644 --- a/doc/build/orm/relationships.rst +++ b/doc/build/orm/relationships.rst @@ -10,1837 +10,14 @@ of its usage. The reference material here continues into the next section, :ref:`collections_toplevel`, which has additional detail on configuration of collections via :func:`relationship`. -.. _relationship_patterns: - -Basic Relational Patterns --------------------------- - -A quick walkthrough of the basic relational patterns. - -The imports used for each of the following sections is as follows:: - - from sqlalchemy import Table, Column, Integer, ForeignKey - from sqlalchemy.orm import relationship, backref - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - -One To Many -~~~~~~~~~~~~ - -A one to many relationship places a foreign key on the child table referencing -the parent. :func:`.relationship` is then specified on the parent, as referencing -a collection of items represented by the child:: - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - children = relationship("Child") - - class Child(Base): - __tablename__ = 'child' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('parent.id')) - -To establish a bidirectional relationship in one-to-many, where the "reverse" -side is a many to one, specify the :paramref:`~.relationship.backref` option:: - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - children = relationship("Child", backref="parent") - - class Child(Base): - __tablename__ = 'child' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('parent.id')) - -``Child`` will get a ``parent`` attribute with many-to-one semantics. - -Many To One -~~~~~~~~~~~~ - -Many to one places a foreign key in the parent table referencing the child. -:func:`.relationship` is declared on the parent, where a new scalar-holding -attribute will be created:: - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - child_id = Column(Integer, ForeignKey('child.id')) - child = relationship("Child") - - class Child(Base): - __tablename__ = 'child' - id = Column(Integer, primary_key=True) - -Bidirectional behavior is achieved by setting -:paramref:`~.relationship.backref` to the value ``"parents"``, which -will place a one-to-many collection on the ``Child`` class:: - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - child_id = Column(Integer, ForeignKey('child.id')) - child = relationship("Child", backref="parents") - -.. _relationships_one_to_one: - -One To One -~~~~~~~~~~~ - -One To One is essentially a bidirectional relationship with a scalar -attribute on both sides. To achieve this, the :paramref:`~.relationship.uselist` flag indicates -the placement of a scalar attribute instead of a collection on the "many" side -of the relationship. To convert one-to-many into one-to-one:: - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - child = relationship("Child", uselist=False, backref="parent") - - class Child(Base): - __tablename__ = 'child' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('parent.id')) - -Or to turn a one-to-many backref into one-to-one, use the :func:`.backref` function -to provide arguments for the reverse side:: - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - child_id = Column(Integer, ForeignKey('child.id')) - child = relationship("Child", backref=backref("parent", uselist=False)) - - class Child(Base): - __tablename__ = 'child' - id = Column(Integer, primary_key=True) - -.. _relationships_many_to_many: - -Many To Many -~~~~~~~~~~~~~ - -Many to Many adds an association table between two classes. The association -table is indicated by the :paramref:`~.relationship.secondary` argument to -:func:`.relationship`. Usually, the :class:`.Table` uses the :class:`.MetaData` -object associated with the declarative base class, so that the :class:`.ForeignKey` -directives can locate the remote tables with which to link:: - - association_table = Table('association', Base.metadata, - Column('left_id', Integer, ForeignKey('left.id')), - Column('right_id', Integer, ForeignKey('right.id')) - ) - - class Parent(Base): - __tablename__ = 'left' - id = Column(Integer, primary_key=True) - children = relationship("Child", - secondary=association_table) - - class Child(Base): - __tablename__ = 'right' - id = Column(Integer, primary_key=True) - -For a bidirectional relationship, both sides of the relationship contain a -collection. The :paramref:`~.relationship.backref` keyword will automatically use -the same :paramref:`~.relationship.secondary` argument for the reverse relationship:: - - association_table = Table('association', Base.metadata, - Column('left_id', Integer, ForeignKey('left.id')), - Column('right_id', Integer, ForeignKey('right.id')) - ) - - class Parent(Base): - __tablename__ = 'left' - id = Column(Integer, primary_key=True) - children = relationship("Child", - secondary=association_table, - backref="parents") - - class Child(Base): - __tablename__ = 'right' - id = Column(Integer, primary_key=True) - -The :paramref:`~.relationship.secondary` argument of :func:`.relationship` also accepts a callable -that returns the ultimate argument, which is evaluated only when mappers are -first used. Using this, we can define the ``association_table`` at a later -point, as long as it's available to the callable after all module initialization -is complete:: - - class Parent(Base): - __tablename__ = 'left' - id = Column(Integer, primary_key=True) - children = relationship("Child", - secondary=lambda: association_table, - backref="parents") - -With the declarative extension in use, the traditional "string name of the table" -is accepted as well, matching the name of the table as stored in ``Base.metadata.tables``:: - - class Parent(Base): - __tablename__ = 'left' - id = Column(Integer, primary_key=True) - children = relationship("Child", - secondary="association", - backref="parents") - -.. _relationships_many_to_many_deletion: - -Deleting Rows from the Many to Many Table -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A behavior which is unique to the :paramref:`~.relationship.secondary` argument to :func:`.relationship` -is that the :class:`.Table` which is specified here is automatically subject -to INSERT and DELETE statements, as objects are added or removed from the collection. -There is **no need to delete from this table manually**. The act of removing a -record from the collection will have the effect of the row being deleted on flush:: - - # row will be deleted from the "secondary" table - # automatically - myparent.children.remove(somechild) - -A question which often arises is how the row in the "secondary" table can be deleted -when the child object is handed directly to :meth:`.Session.delete`:: - - session.delete(somechild) - -There are several possibilities here: - -* If there is a :func:`.relationship` from ``Parent`` to ``Child``, but there is - **not** a reverse-relationship that links a particular ``Child`` to each ``Parent``, - SQLAlchemy will not have any awareness that when deleting this particular - ``Child`` object, it needs to maintain the "secondary" table that links it to - the ``Parent``. No delete of the "secondary" table will occur. -* If there is a relationship that links a particular ``Child`` to each ``Parent``, - suppose it's called ``Child.parents``, SQLAlchemy by default will load in - the ``Child.parents`` collection to locate all ``Parent`` objects, and remove - each row from the "secondary" table which establishes this link. Note that - this relationship does not need to be bidrectional; SQLAlchemy is strictly - looking at every :func:`.relationship` associated with the ``Child`` object - being deleted. -* A higher performing option here is to use ON DELETE CASCADE directives - with the foreign keys used by the database. Assuming the database supports - this feature, the database itself can be made to automatically delete rows in the - "secondary" table as referencing rows in "child" are deleted. SQLAlchemy - can be instructed to forego actively loading in the ``Child.parents`` - collection in this case using the :paramref:`~.relationship.passive_deletes` - directive on :func:`.relationship`; see :ref:`passive_deletes` for more details - on this. - -Note again, these behaviors are *only* relevant to the :paramref:`~.relationship.secondary` option -used with :func:`.relationship`. If dealing with association tables that -are mapped explicitly and are *not* present in the :paramref:`~.relationship.secondary` option -of a relevant :func:`.relationship`, cascade rules can be used instead -to automatically delete entities in reaction to a related entity being -deleted - see :ref:`unitofwork_cascades` for information on this feature. - - -.. _association_pattern: - -Association Object -~~~~~~~~~~~~~~~~~~ - -The association object pattern is a variant on many-to-many: it's used -when your association table contains additional columns beyond those -which are foreign keys to the left and right tables. Instead of using -the :paramref:`~.relationship.secondary` argument, you map a new class -directly to the association table. The left side of the relationship -references the association object via one-to-many, and the association -class references the right side via many-to-one. Below we illustrate -an association table mapped to the ``Association`` class which -includes a column called ``extra_data``, which is a string value that -is stored along with each association between ``Parent`` and -``Child``:: - - class Association(Base): - __tablename__ = 'association' - left_id = Column(Integer, ForeignKey('left.id'), primary_key=True) - right_id = Column(Integer, ForeignKey('right.id'), primary_key=True) - extra_data = Column(String(50)) - child = relationship("Child") - - class Parent(Base): - __tablename__ = 'left' - id = Column(Integer, primary_key=True) - children = relationship("Association") - - class Child(Base): - __tablename__ = 'right' - id = Column(Integer, primary_key=True) - -The bidirectional version adds backrefs to both relationships:: - - class Association(Base): - __tablename__ = 'association' - left_id = Column(Integer, ForeignKey('left.id'), primary_key=True) - right_id = Column(Integer, ForeignKey('right.id'), primary_key=True) - extra_data = Column(String(50)) - child = relationship("Child", backref="parent_assocs") - - class Parent(Base): - __tablename__ = 'left' - id = Column(Integer, primary_key=True) - children = relationship("Association", backref="parent") - - class Child(Base): - __tablename__ = 'right' - id = Column(Integer, primary_key=True) - -Working with the association pattern in its direct form requires that child -objects are associated with an association instance before being appended to -the parent; similarly, access from parent to child goes through the -association object:: - - # create parent, append a child via association - p = Parent() - a = Association(extra_data="some data") - a.child = Child() - p.children.append(a) - - # iterate through child objects via association, including association - # attributes - for assoc in p.children: - print assoc.extra_data - print assoc.child - -To enhance the association object pattern such that direct -access to the ``Association`` object is optional, SQLAlchemy -provides the :ref:`associationproxy_toplevel` extension. This -extension allows the configuration of attributes which will -access two "hops" with a single access, one "hop" to the -associated object, and a second to a target attribute. - -.. note:: - - When using the association object pattern, it is advisable that the - association-mapped table not be used as the - :paramref:`~.relationship.secondary` argument on a - :func:`.relationship` elsewhere, unless that :func:`.relationship` - contains the option :paramref:`~.relationship.viewonly` set to - ``True``. SQLAlchemy otherwise may attempt to emit redundant INSERT - and DELETE statements on the same table, if similar state is - detected on the related attribute as well as the associated object. - -.. _self_referential: - -Adjacency List Relationships ------------------------------ - -The **adjacency list** pattern is a common relational pattern whereby a table -contains a foreign key reference to itself. This is the most common -way to represent hierarchical data in flat tables. Other methods -include **nested sets**, sometimes called "modified preorder", -as well as **materialized path**. Despite the appeal that modified preorder -has when evaluated for its fluency within SQL queries, the adjacency list model is -probably the most appropriate pattern for the large majority of hierarchical -storage needs, for reasons of concurrency, reduced complexity, and that -modified preorder has little advantage over an application which can fully -load subtrees into the application space. - -In this example, we'll work with a single mapped -class called ``Node``, representing a tree structure:: - - class Node(Base): - __tablename__ = 'node' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('node.id')) - data = Column(String(50)) - children = relationship("Node") - -With this structure, a graph such as the following:: - - root --+---> child1 - +---> child2 --+--> subchild1 - | +--> subchild2 - +---> child3 - -Would be represented with data such as:: - - id parent_id data - --- ------- ---- - 1 NULL root - 2 1 child1 - 3 1 child2 - 4 3 subchild1 - 5 3 subchild2 - 6 1 child3 - -The :func:`.relationship` configuration here works in the -same way as a "normal" one-to-many relationship, with the -exception that the "direction", i.e. whether the relationship -is one-to-many or many-to-one, is assumed by default to -be one-to-many. To establish the relationship as many-to-one, -an extra directive is added known as :paramref:`~.relationship.remote_side`, which -is a :class:`.Column` or collection of :class:`.Column` objects -that indicate those which should be considered to be "remote":: - - class Node(Base): - __tablename__ = 'node' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('node.id')) - data = Column(String(50)) - parent = relationship("Node", remote_side=[id]) - -Where above, the ``id`` column is applied as the :paramref:`~.relationship.remote_side` -of the ``parent`` :func:`.relationship`, thus establishing -``parent_id`` as the "local" side, and the relationship -then behaves as a many-to-one. - -As always, both directions can be combined into a bidirectional -relationship using the :func:`.backref` function:: - - class Node(Base): - __tablename__ = 'node' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('node.id')) - data = Column(String(50)) - children = relationship("Node", - backref=backref('parent', remote_side=[id]) - ) - -There are several examples included with SQLAlchemy illustrating -self-referential strategies; these include :ref:`examples_adjacencylist` and -:ref:`examples_xmlpersistence`. - -Composite Adjacency Lists -~~~~~~~~~~~~~~~~~~~~~~~~~ - -A sub-category of the adjacency list relationship is the rare -case where a particular column is present on both the "local" and -"remote" side of the join condition. An example is the ``Folder`` -class below; using a composite primary key, the ``account_id`` -column refers to itself, to indicate sub folders which are within -the same account as that of the parent; while ``folder_id`` refers -to a specific folder within that account:: - - class Folder(Base): - __tablename__ = 'folder' - __table_args__ = ( - ForeignKeyConstraint( - ['account_id', 'parent_id'], - ['folder.account_id', 'folder.folder_id']), - ) - - account_id = Column(Integer, primary_key=True) - folder_id = Column(Integer, primary_key=True) - parent_id = Column(Integer) - name = Column(String) - - parent_folder = relationship("Folder", - backref="child_folders", - remote_side=[account_id, folder_id] - ) - -Above, we pass ``account_id`` into the :paramref:`~.relationship.remote_side` list. -:func:`.relationship` recognizes that the ``account_id`` column here -is on both sides, and aligns the "remote" column along with the -``folder_id`` column, which it recognizes as uniquely present on -the "remote" side. - -.. versionadded:: 0.8 - Support for self-referential composite keys in :func:`.relationship` - where a column points to itself. - -Self-Referential Query Strategies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Querying of self-referential structures works like any other query:: - - # get all nodes named 'child2' - session.query(Node).filter(Node.data=='child2') - -However extra care is needed when attempting to join along -the foreign key from one level of the tree to the next. In SQL, -a join from a table to itself requires that at least one side of the -expression be "aliased" so that it can be unambiguously referred to. - -Recall from :ref:`ormtutorial_aliases` in the ORM tutorial that the -:func:`.orm.aliased` construct is normally used to provide an "alias" of -an ORM entity. Joining from ``Node`` to itself using this technique -looks like: - -.. sourcecode:: python+sql - - from sqlalchemy.orm import aliased - - nodealias = aliased(Node) - {sql}session.query(Node).filter(Node.data=='subchild1').\ - join(nodealias, Node.parent).\ - filter(nodealias.data=="child2").\ - all() - SELECT node.id AS node_id, - node.parent_id AS node_parent_id, - node.data AS node_data - FROM node JOIN node AS node_1 - ON node.parent_id = node_1.id - WHERE node.data = ? - AND node_1.data = ? - ['subchild1', 'child2'] - -:meth:`.Query.join` also includes a feature known as -:paramref:`.Query.join.aliased` that can shorten the verbosity self- -referential joins, at the expense of query flexibility. This feature -performs a similar "aliasing" step to that above, without the need for -an explicit entity. Calls to :meth:`.Query.filter` and similar -subsequent to the aliased join will **adapt** the ``Node`` entity to -be that of the alias: - -.. sourcecode:: python+sql - - {sql}session.query(Node).filter(Node.data=='subchild1').\ - join(Node.parent, aliased=True).\ - filter(Node.data=='child2').\ - all() - SELECT node.id AS node_id, - node.parent_id AS node_parent_id, - node.data AS node_data - FROM node - JOIN node AS node_1 ON node_1.id = node.parent_id - WHERE node.data = ? AND node_1.data = ? - ['subchild1', 'child2'] - -To add criterion to multiple points along a longer join, add -:paramref:`.Query.join.from_joinpoint` to the additional -:meth:`~.Query.join` calls: - -.. sourcecode:: python+sql - - # get all nodes named 'subchild1' with a - # parent named 'child2' and a grandparent 'root' - {sql}session.query(Node).\ - filter(Node.data=='subchild1').\ - join(Node.parent, aliased=True).\ - filter(Node.data=='child2').\ - join(Node.parent, aliased=True, from_joinpoint=True).\ - filter(Node.data=='root').\ - all() - SELECT node.id AS node_id, - node.parent_id AS node_parent_id, - node.data AS node_data - FROM node - JOIN node AS node_1 ON node_1.id = node.parent_id - JOIN node AS node_2 ON node_2.id = node_1.parent_id - WHERE node.data = ? - AND node_1.data = ? - AND node_2.data = ? - ['subchild1', 'child2', 'root'] - -:meth:`.Query.reset_joinpoint` will also remove the "aliasing" from filtering -calls:: - - session.query(Node).\ - join(Node.children, aliased=True).\ - filter(Node.data == 'foo').\ - reset_joinpoint().\ - filter(Node.data == 'bar') - -For an example of using :paramref:`.Query.join.aliased` to -arbitrarily join along a chain of self-referential nodes, see -:ref:`examples_xmlpersistence`. - -.. _self_referential_eager_loading: - -Configuring Self-Referential Eager Loading -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Eager loading of relationships occurs using joins or outerjoins from parent to -child table during a normal query operation, such that the parent and its -immediate child collection or reference can be populated from a single SQL -statement, or a second statement for all immediate child collections. -SQLAlchemy's joined and subquery eager loading use aliased tables in all cases -when joining to related items, so are compatible with self-referential -joining. However, to use eager loading with a self-referential relationship, -SQLAlchemy needs to be told how many levels deep it should join and/or query; -otherwise the eager load will not take place at all. This depth setting is -configured via :paramref:`~.relationships.join_depth`: - -.. sourcecode:: python+sql - - class Node(Base): - __tablename__ = 'node' - id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('node.id')) - data = Column(String(50)) - children = relationship("Node", - lazy="joined", - join_depth=2) - - {sql}session.query(Node).all() - SELECT node_1.id AS node_1_id, - node_1.parent_id AS node_1_parent_id, - node_1.data AS node_1_data, - node_2.id AS node_2_id, - node_2.parent_id AS node_2_parent_id, - node_2.data AS node_2_data, - node.id AS node_id, - node.parent_id AS node_parent_id, - node.data AS node_data - FROM node - LEFT OUTER JOIN node AS node_2 - ON node.id = node_2.parent_id - LEFT OUTER JOIN node AS node_1 - ON node_2.id = node_1.parent_id - [] - -.. _relationships_backref: - -Linking Relationships with Backref ----------------------------------- - -The :paramref:`~.relationship.backref` keyword argument was first introduced in :ref:`ormtutorial_toplevel`, and has been -mentioned throughout many of the examples here. What does it actually do ? Let's start -with the canonical ``User`` and ``Address`` scenario:: - - from sqlalchemy import Integer, ForeignKey, String, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - name = Column(String) - - addresses = relationship("Address", backref="user") - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - email = Column(String) - user_id = Column(Integer, ForeignKey('user.id')) - -The above configuration establishes a collection of ``Address`` objects on ``User`` called -``User.addresses``. It also establishes a ``.user`` attribute on ``Address`` which will -refer to the parent ``User`` object. - -In fact, the :paramref:`~.relationship.backref` keyword is only a common shortcut for placing a second -:func:`.relationship` onto the ``Address`` mapping, including the establishment -of an event listener on both sides which will mirror attribute operations -in both directions. The above configuration is equivalent to:: - - from sqlalchemy import Integer, ForeignKey, String, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - name = Column(String) - - addresses = relationship("Address", back_populates="user") - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - email = Column(String) - user_id = Column(Integer, ForeignKey('user.id')) - - user = relationship("User", back_populates="addresses") - -Above, we add a ``.user`` relationship to ``Address`` explicitly. On -both relationships, the :paramref:`~.relationship.back_populates` directive tells each relationship -about the other one, indicating that they should establish "bidirectional" -behavior between each other. The primary effect of this configuration -is that the relationship adds event handlers to both attributes -which have the behavior of "when an append or set event occurs here, set ourselves -onto the incoming attribute using this particular attribute name". -The behavior is illustrated as follows. Start with a ``User`` and an ``Address`` -instance. The ``.addresses`` collection is empty, and the ``.user`` attribute -is ``None``:: - - >>> u1 = User() - >>> a1 = Address() - >>> u1.addresses - [] - >>> print a1.user - None - -However, once the ``Address`` is appended to the ``u1.addresses`` collection, -both the collection and the scalar attribute have been populated:: - - >>> u1.addresses.append(a1) - >>> u1.addresses - [<__main__.Address object at 0x12a6ed0>] - >>> a1.user - <__main__.User object at 0x12a6590> - -This behavior of course works in reverse for removal operations as well, as well -as for equivalent operations on both sides. Such as -when ``.user`` is set again to ``None``, the ``Address`` object is removed -from the reverse collection:: - - >>> a1.user = None - >>> u1.addresses - [] - -The manipulation of the ``.addresses`` collection and the ``.user`` attribute -occurs entirely in Python without any interaction with the SQL database. -Without this behavior, the proper state would be apparent on both sides once the -data has been flushed to the database, and later reloaded after a commit or -expiration operation occurs. The :paramref:`~.relationship.backref`/:paramref:`~.relationship.back_populates` behavior has the advantage -that common bidirectional operations can reflect the correct state without requiring -a database round trip. - -Remember, when the :paramref:`~.relationship.backref` keyword is used on a single relationship, it's -exactly the same as if the above two relationships were created individually -using :paramref:`~.relationship.back_populates` on each. - -Backref Arguments -~~~~~~~~~~~~~~~~~~ - -We've established that the :paramref:`~.relationship.backref` keyword is merely a shortcut for building -two individual :func:`.relationship` constructs that refer to each other. Part of -the behavior of this shortcut is that certain configurational arguments applied to -the :func:`.relationship` -will also be applied to the other direction - namely those arguments that describe -the relationship at a schema level, and are unlikely to be different in the reverse -direction. The usual case -here is a many-to-many :func:`.relationship` that has a :paramref:`~.relationship.secondary` argument, -or a one-to-many or many-to-one which has a :paramref:`~.relationship.primaryjoin` argument (the -:paramref:`~.relationship.primaryjoin` argument is discussed in :ref:`relationship_primaryjoin`). Such -as if we limited the list of ``Address`` objects to those which start with "tony":: - - from sqlalchemy import Integer, ForeignKey, String, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - name = Column(String) - - addresses = relationship("Address", - primaryjoin="and_(User.id==Address.user_id, " - "Address.email.startswith('tony'))", - backref="user") - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - email = Column(String) - user_id = Column(Integer, ForeignKey('user.id')) - -We can observe, by inspecting the resulting property, that both sides -of the relationship have this join condition applied:: - - >>> print User.addresses.property.primaryjoin - "user".id = address.user_id AND address.email LIKE :email_1 || '%%' - >>> - >>> print Address.user.property.primaryjoin - "user".id = address.user_id AND address.email LIKE :email_1 || '%%' - >>> - -This reuse of arguments should pretty much do the "right thing" - it -uses only arguments that are applicable, and in the case of a many-to- -many relationship, will reverse the usage of -:paramref:`~.relationship.primaryjoin` and -:paramref:`~.relationship.secondaryjoin` to correspond to the other -direction (see the example in :ref:`self_referential_many_to_many` for -this). - -It's very often the case however that we'd like to specify arguments -that are specific to just the side where we happened to place the -"backref". This includes :func:`.relationship` arguments like -:paramref:`~.relationship.lazy`, -:paramref:`~.relationship.remote_side`, -:paramref:`~.relationship.cascade` and -:paramref:`~.relationship.cascade_backrefs`. For this case we use -the :func:`.backref` function in place of a string:: - - # - from sqlalchemy.orm import backref - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - name = Column(String) - - addresses = relationship("Address", - backref=backref("user", lazy="joined")) - -Where above, we placed a ``lazy="joined"`` directive only on the ``Address.user`` -side, indicating that when a query against ``Address`` is made, a join to the ``User`` -entity should be made automatically which will populate the ``.user`` attribute of each -returned ``Address``. The :func:`.backref` function formatted the arguments we gave -it into a form that is interpreted by the receiving :func:`.relationship` as additional -arguments to be applied to the new relationship it creates. - -One Way Backrefs -~~~~~~~~~~~~~~~~~ - -An unusual case is that of the "one way backref". This is where the -"back-populating" behavior of the backref is only desirable in one -direction. An example of this is a collection which contains a -filtering :paramref:`~.relationship.primaryjoin` condition. We'd -like to append items to this collection as needed, and have them -populate the "parent" object on the incoming object. However, we'd -also like to have items that are not part of the collection, but still -have the same "parent" association - these items should never be in -the collection. - -Taking our previous example, where we established a -:paramref:`~.relationship.primaryjoin` that limited the collection -only to ``Address`` objects whose email address started with the word -``tony``, the usual backref behavior is that all items populate in -both directions. We wouldn't want this behavior for a case like the -following:: - - >>> u1 = User() - >>> a1 = Address(email='mary') - >>> a1.user = u1 - >>> u1.addresses - [<__main__.Address object at 0x1411910>] - -Above, the ``Address`` object that doesn't match the criterion of "starts with 'tony'" -is present in the ``addresses`` collection of ``u1``. After these objects are flushed, -the transaction committed and their attributes expired for a re-load, the ``addresses`` -collection will hit the database on next access and no longer have this ``Address`` object -present, due to the filtering condition. But we can do away with this unwanted side -of the "backref" behavior on the Python side by using two separate :func:`.relationship` constructs, -placing :paramref:`~.relationship.back_populates` only on one side:: - - from sqlalchemy import Integer, ForeignKey, String, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - name = Column(String) - addresses = relationship("Address", - primaryjoin="and_(User.id==Address.user_id, " - "Address.email.startswith('tony'))", - back_populates="user") - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - email = Column(String) - user_id = Column(Integer, ForeignKey('user.id')) - user = relationship("User") - -With the above scenario, appending an ``Address`` object to the ``.addresses`` -collection of a ``User`` will always establish the ``.user`` attribute on that -``Address``:: - - >>> u1 = User() - >>> a1 = Address(email='tony') - >>> u1.addresses.append(a1) - >>> a1.user - <__main__.User object at 0x1411850> - -However, applying a ``User`` to the ``.user`` attribute of an ``Address``, -will not append the ``Address`` object to the collection:: - - >>> a2 = Address(email='mary') - >>> a2.user = u1 - >>> a2 in u1.addresses - False - -Of course, we've disabled some of the usefulness of -:paramref:`~.relationship.backref` here, in that when we do append an -``Address`` that corresponds to the criteria of -``email.startswith('tony')``, it won't show up in the -``User.addresses`` collection until the session is flushed, and the -attributes reloaded after a commit or expire operation. While we -could consider an attribute event that checks this criterion in -Python, this starts to cross the line of duplicating too much SQL -behavior in Python. The backref behavior itself is only a slight -transgression of this philosophy - SQLAlchemy tries to keep these to a -minimum overall. - -.. _relationship_configure_joins: - -Configuring how Relationship Joins ------------------------------------- - -:func:`.relationship` will normally create a join between two tables -by examining the foreign key relationship between the two tables -to determine which columns should be compared. There are a variety -of situations where this behavior needs to be customized. - -.. _relationship_foreign_keys: - -Handling Multiple Join Paths -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One of the most common situations to deal with is when -there are more than one foreign key path between two tables. - -Consider a ``Customer`` class that contains two foreign keys to an ``Address`` -class:: - - from sqlalchemy import Integer, ForeignKey, String, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class Customer(Base): - __tablename__ = 'customer' - id = Column(Integer, primary_key=True) - name = Column(String) - - billing_address_id = Column(Integer, ForeignKey("address.id")) - shipping_address_id = Column(Integer, ForeignKey("address.id")) - - billing_address = relationship("Address") - shipping_address = relationship("Address") - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - street = Column(String) - city = Column(String) - state = Column(String) - zip = Column(String) - -The above mapping, when we attempt to use it, will produce the error:: - - sqlalchemy.exc.AmbiguousForeignKeysError: Could not determine join - condition between parent/child tables on relationship - Customer.billing_address - there are multiple foreign key - paths linking the tables. Specify the 'foreign_keys' argument, - providing a list of those columns which should be - counted as containing a foreign key reference to the parent table. - -The above message is pretty long. There are many potential messages -that :func:`.relationship` can return, which have been carefully tailored -to detect a variety of common configurational issues; most will suggest -the additional configuration that's needed to resolve the ambiguity -or other missing information. - -In this case, the message wants us to qualify each :func:`.relationship` -by instructing for each one which foreign key column should be considered, and -the appropriate form is as follows:: - - class Customer(Base): - __tablename__ = 'customer' - id = Column(Integer, primary_key=True) - name = Column(String) - - billing_address_id = Column(Integer, ForeignKey("address.id")) - shipping_address_id = Column(Integer, ForeignKey("address.id")) - - billing_address = relationship("Address", foreign_keys=[billing_address_id]) - shipping_address = relationship("Address", foreign_keys=[shipping_address_id]) - -Above, we specify the ``foreign_keys`` argument, which is a :class:`.Column` or list -of :class:`.Column` objects which indicate those columns to be considered "foreign", -or in other words, the columns that contain a value referring to a parent table. -Loading the ``Customer.billing_address`` relationship from a ``Customer`` -object will use the value present in ``billing_address_id`` in order to -identify the row in ``Address`` to be loaded; similarly, ``shipping_address_id`` -is used for the ``shipping_address`` relationship. The linkage of the two -columns also plays a role during persistence; the newly generated primary key -of a just-inserted ``Address`` object will be copied into the appropriate -foreign key column of an associated ``Customer`` object during a flush. - -When specifying ``foreign_keys`` with Declarative, we can also use string -names to specify, however it is important that if using a list, the **list -is part of the string**:: - - billing_address = relationship("Address", foreign_keys="[Customer.billing_address_id]") - -In this specific example, the list is not necessary in any case as there's only -one :class:`.Column` we need:: - - billing_address = relationship("Address", foreign_keys="Customer.billing_address_id") - -.. versionchanged:: 0.8 - :func:`.relationship` can resolve ambiguity between foreign key targets on the - basis of the ``foreign_keys`` argument alone; the :paramref:`~.relationship.primaryjoin` - argument is no longer needed in this situation. - -.. _relationship_primaryjoin: - -Specifying Alternate Join Conditions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The default behavior of :func:`.relationship` when constructing a join -is that it equates the value of primary key columns -on one side to that of foreign-key-referring columns on the other. -We can change this criterion to be anything we'd like using the -:paramref:`~.relationship.primaryjoin` -argument, as well as the :paramref:`~.relationship.secondaryjoin` -argument in the case when a "secondary" table is used. - -In the example below, using the ``User`` class -as well as an ``Address`` class which stores a street address, we -create a relationship ``boston_addresses`` which will only -load those ``Address`` objects which specify a city of "Boston":: - - from sqlalchemy import Integer, ForeignKey, String, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - name = Column(String) - boston_addresses = relationship("Address", - primaryjoin="and_(User.id==Address.user_id, " - "Address.city=='Boston')") - - class Address(Base): - __tablename__ = 'address' - id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('user.id')) - - street = Column(String) - city = Column(String) - state = Column(String) - zip = Column(String) - -Within this string SQL expression, we made use of the :func:`.and_` conjunction construct to establish -two distinct predicates for the join condition - joining both the ``User.id`` and -``Address.user_id`` columns to each other, as well as limiting rows in ``Address`` -to just ``city='Boston'``. When using Declarative, rudimentary SQL functions like -:func:`.and_` are automatically available in the evaluated namespace of a string -:func:`.relationship` argument. - -The custom criteria we use in a :paramref:`~.relationship.primaryjoin` -is generally only significant when SQLAlchemy is rendering SQL in -order to load or represent this relationship. That is, it's used in -the SQL statement that's emitted in order to perform a per-attribute -lazy load, or when a join is constructed at query time, such as via -:meth:`.Query.join`, or via the eager "joined" or "subquery" styles of -loading. When in-memory objects are being manipulated, we can place -any ``Address`` object we'd like into the ``boston_addresses`` -collection, regardless of what the value of the ``.city`` attribute -is. The objects will remain present in the collection until the -attribute is expired and re-loaded from the database where the -criterion is applied. When a flush occurs, the objects inside of -``boston_addresses`` will be flushed unconditionally, assigning value -of the primary key ``user.id`` column onto the foreign-key-holding -``address.user_id`` column for each row. The ``city`` criteria has no -effect here, as the flush process only cares about synchronizing -primary key values into referencing foreign key values. - -.. _relationship_custom_foreign: - -Creating Custom Foreign Conditions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Another element of the primary join condition is how those columns -considered "foreign" are determined. Usually, some subset -of :class:`.Column` objects will specify :class:`.ForeignKey`, or otherwise -be part of a :class:`.ForeignKeyConstraint` that's relevant to the join condition. -:func:`.relationship` looks to this foreign key status as it decides -how it should load and persist data for this relationship. However, the -:paramref:`~.relationship.primaryjoin` argument can be used to create a join condition that -doesn't involve any "schema" level foreign keys. We can combine :paramref:`~.relationship.primaryjoin` -along with :paramref:`~.relationship.foreign_keys` and :paramref:`~.relationship.remote_side` explicitly in order to -establish such a join. - -Below, a class ``HostEntry`` joins to itself, equating the string ``content`` -column to the ``ip_address`` column, which is a Postgresql type called ``INET``. -We need to use :func:`.cast` in order to cast one side of the join to the -type of the other:: - - from sqlalchemy import cast, String, Column, Integer - from sqlalchemy.orm import relationship - from sqlalchemy.dialects.postgresql import INET - - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class HostEntry(Base): - __tablename__ = 'host_entry' - - id = Column(Integer, primary_key=True) - ip_address = Column(INET) - content = Column(String(50)) - - # relationship() using explicit foreign_keys, remote_side - parent_host = relationship("HostEntry", - primaryjoin=ip_address == cast(content, INET), - foreign_keys=content, - remote_side=ip_address - ) - -The above relationship will produce a join like:: - - SELECT host_entry.id, host_entry.ip_address, host_entry.content - FROM host_entry JOIN host_entry AS host_entry_1 - ON host_entry_1.ip_address = CAST(host_entry.content AS INET) - -An alternative syntax to the above is to use the :func:`.foreign` and -:func:`.remote` :term:`annotations`, -inline within the :paramref:`~.relationship.primaryjoin` expression. -This syntax represents the annotations that :func:`.relationship` normally -applies by itself to the join condition given the :paramref:`~.relationship.foreign_keys` and -:paramref:`~.relationship.remote_side` arguments. These functions may -be more succinct when an explicit join condition is present, and additionally -serve to mark exactly the column that is "foreign" or "remote" independent -of whether that column is stated multiple times or within complex -SQL expressions:: - - from sqlalchemy.orm import foreign, remote - - class HostEntry(Base): - __tablename__ = 'host_entry' - - id = Column(Integer, primary_key=True) - ip_address = Column(INET) - content = Column(String(50)) - - # relationship() using explicit foreign() and remote() annotations - # in lieu of separate arguments - parent_host = relationship("HostEntry", - primaryjoin=remote(ip_address) == \ - cast(foreign(content), INET), - ) - - -.. _relationship_custom_operator: - -Using custom operators in join conditions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Another use case for relationships is the use of custom operators, such -as Postgresql's "is contained within" ``<<`` operator when joining with -types such as :class:`.postgresql.INET` and :class:`.postgresql.CIDR`. -For custom operators we use the :meth:`.Operators.op` function:: - - inet_column.op("<<")(cidr_column) - -However, if we construct a :paramref:`~.relationship.primaryjoin` using this -operator, :func:`.relationship` will still need more information. This is because -when it examines our primaryjoin condition, it specifically looks for operators -used for **comparisons**, and this is typically a fixed list containing known -comparison operators such as ``==``, ``<``, etc. So for our custom operator -to participate in this system, we need it to register as a comparison operator -using the :paramref:`~.Operators.op.is_comparison` parameter:: - - inet_column.op("<<", is_comparison=True)(cidr_column) - -A complete example:: - - class IPA(Base): - __tablename__ = 'ip_address' - - id = Column(Integer, primary_key=True) - v4address = Column(INET) - - network = relationship("Network", - primaryjoin="IPA.v4address.op('<<', is_comparison=True)" - "(foreign(Network.v4representation))", - viewonly=True - ) - class Network(Base): - __tablename__ = 'network' - - id = Column(Integer, primary_key=True) - v4representation = Column(CIDR) - -Above, a query such as:: - - session.query(IPA).join(IPA.network) - -Will render as:: - - SELECT ip_address.id AS ip_address_id, ip_address.v4address AS ip_address_v4address - FROM ip_address JOIN network ON ip_address.v4address << network.v4representation - -.. versionadded:: 0.9.2 - Added the :paramref:`.Operators.op.is_comparison` - flag to assist in the creation of :func:`.relationship` constructs using - custom operators. - -.. _relationship_overlapping_foreignkeys: - -Overlapping Foreign Keys -~~~~~~~~~~~~~~~~~~~~~~~~ - -A rare scenario can arise when composite foreign keys are used, such that -a single column may be the subject of more than one column -referred to via foreign key constraint. - -Consider an (admittedly complex) mapping such as the ``Magazine`` object, -referred to both by the ``Writer`` object and the ``Article`` object -using a composite primary key scheme that includes ``magazine_id`` -for both; then to make ``Article`` refer to ``Writer`` as well, -``Article.magazine_id`` is involved in two separate relationships; -``Article.magazine`` and ``Article.writer``:: - - class Magazine(Base): - __tablename__ = 'magazine' - - id = Column(Integer, primary_key=True) - - - class Article(Base): - __tablename__ = 'article' - - article_id = Column(Integer) - magazine_id = Column(ForeignKey('magazine.id')) - writer_id = Column() - - magazine = relationship("Magazine") - writer = relationship("Writer") - - __table_args__ = ( - PrimaryKeyConstraint('article_id', 'magazine_id'), - ForeignKeyConstraint( - ['writer_id', 'magazine_id'], - ['writer.id', 'writer.magazine_id'] - ), - ) - - - class Writer(Base): - __tablename__ = 'writer' - - id = Column(Integer, primary_key=True) - magazine_id = Column(ForeignKey('magazine.id'), primary_key=True) - magazine = relationship("Magazine") - -When the above mapping is configured, we will see this warning emitted:: - - SAWarning: relationship 'Article.writer' will copy column - writer.magazine_id to column article.magazine_id, - which conflicts with relationship(s): 'Article.magazine' - (copies magazine.id to article.magazine_id). Consider applying - viewonly=True to read-only relationships, or provide a primaryjoin - condition marking writable columns with the foreign() annotation. - -What this refers to originates from the fact that ``Article.magazine_id`` is -the subject of two different foreign key constraints; it refers to -``Magazine.id`` directly as a source column, but also refers to -``Writer.magazine_id`` as a source column in the context of the -composite key to ``Writer``. If we associate an ``Article`` with a -particular ``Magazine``, but then associate the ``Article`` with a -``Writer`` that's associated with a *different* ``Magazine``, the ORM -will overwrite ``Article.magazine_id`` non-deterministically, silently -changing which magazine we refer towards; it may -also attempt to place NULL into this columnn if we de-associate a -``Writer`` from an ``Article``. The warning lets us know this is the case. - -To solve this, we need to break out the behavior of ``Article`` to include -all three of the following features: - -1. ``Article`` first and foremost writes to - ``Article.magazine_id`` based on data persisted in the ``Article.magazine`` - relationship only, that is a value copied from ``Magazine.id``. - -2. ``Article`` can write to ``Article.writer_id`` on behalf of data - persisted in the ``Article.writer`` relationship, but only the - ``Writer.id`` column; the ``Writer.magazine_id`` column should not - be written into ``Article.magazine_id`` as it ultimately is sourced - from ``Magazine.id``. - -3. ``Article`` takes ``Article.magazine_id`` into account when loading - ``Article.writer``, even though it *doesn't* write to it on behalf - of this relationship. - -To get just #1 and #2, we could specify only ``Article.writer_id`` as the -"foreign keys" for ``Article.writer``:: - - class Article(Base): - # ... - - writer = relationship("Writer", foreign_keys='Article.writer_id') - -However, this has the effect of ``Article.writer`` not taking -``Article.magazine_id`` into account when querying against ``Writer``: - -.. sourcecode:: sql - - SELECT article.article_id AS article_article_id, - article.magazine_id AS article_magazine_id, - article.writer_id AS article_writer_id - FROM article - JOIN writer ON writer.id = article.writer_id - -Therefore, to get at all of #1, #2, and #3, we express the join condition -as well as which columns to be written by combining -:paramref:`~.relationship.primaryjoin` fully, along with either the -:paramref:`~.relationship.foreign_keys` argument, or more succinctly by -annotating with :func:`~.orm.foreign`:: - - class Article(Base): - # ... - - writer = relationship( - "Writer", - primaryjoin="and_(Writer.id == foreign(Article.writer_id), " - "Writer.magazine_id == Article.magazine_id)") - -.. versionchanged:: 1.0.0 the ORM will attempt to warn when a column is used - as the synchronization target from more than one relationship - simultaneously. - - -Non-relational Comparisons / Materialized Path -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. warning:: this section details an experimental feature. - -Using custom expressions means we can produce unorthodox join conditions that -don't obey the usual primary/foreign key model. One such example is the -materialized path pattern, where we compare strings for overlapping path tokens -in order to produce a tree structure. - -Through careful use of :func:`.foreign` and :func:`.remote`, we can build -a relationship that effectively produces a rudimentary materialized path -system. Essentially, when :func:`.foreign` and :func:`.remote` are -on the *same* side of the comparison expression, the relationship is considered -to be "one to many"; when they are on *different* sides, the relationship -is considered to be "many to one". For the comparison we'll use here, -we'll be dealing with collections so we keep things configured as "one to many":: - - class Element(Base): - __tablename__ = 'element' - - path = Column(String, primary_key=True) - - descendants = relationship('Element', - primaryjoin= - remote(foreign(path)).like( - path.concat('/%')), - viewonly=True, - order_by=path) - -Above, if given an ``Element`` object with a path attribute of ``"/foo/bar2"``, -we seek for a load of ``Element.descendants`` to look like:: - - SELECT element.path AS element_path - FROM element - WHERE element.path LIKE ('/foo/bar2' || '/%') ORDER BY element.path - -.. versionadded:: 0.9.5 Support has been added to allow a single-column - comparison to itself within a primaryjoin condition, as well as for - primaryjoin conditions that use :meth:`.Operators.like` as the comparison - operator. - -.. _self_referential_many_to_many: - -Self-Referential Many-to-Many Relationship -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Many to many relationships can be customized by one or both of :paramref:`~.relationship.primaryjoin` -and :paramref:`~.relationship.secondaryjoin` - the latter is significant for a relationship that -specifies a many-to-many reference using the :paramref:`~.relationship.secondary` argument. -A common situation which involves the usage of :paramref:`~.relationship.primaryjoin` and :paramref:`~.relationship.secondaryjoin` -is when establishing a many-to-many relationship from a class to itself, as shown below:: - - from sqlalchemy import Integer, ForeignKey, String, Column, Table - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - node_to_node = Table("node_to_node", Base.metadata, - Column("left_node_id", Integer, ForeignKey("node.id"), primary_key=True), - Column("right_node_id", Integer, ForeignKey("node.id"), primary_key=True) - ) - - class Node(Base): - __tablename__ = 'node' - id = Column(Integer, primary_key=True) - label = Column(String) - right_nodes = relationship("Node", - secondary=node_to_node, - primaryjoin=id==node_to_node.c.left_node_id, - secondaryjoin=id==node_to_node.c.right_node_id, - backref="left_nodes" - ) - -Where above, SQLAlchemy can't know automatically which columns should connect -to which for the ``right_nodes`` and ``left_nodes`` relationships. The :paramref:`~.relationship.primaryjoin` -and :paramref:`~.relationship.secondaryjoin` arguments establish how we'd like to join to the association table. -In the Declarative form above, as we are declaring these conditions within the Python -block that corresponds to the ``Node`` class, the ``id`` variable is available directly -as the :class:`.Column` object we wish to join with. - -Alternatively, we can define the :paramref:`~.relationship.primaryjoin` -and :paramref:`~.relationship.secondaryjoin` arguments using strings, which is suitable -in the case that our configuration does not have either the ``Node.id`` column -object available yet or the ``node_to_node`` table perhaps isn't yet available. -When referring to a plain :class:`.Table` object in a declarative string, we -use the string name of the table as it is present in the :class:`.MetaData`:: - - class Node(Base): - __tablename__ = 'node' - id = Column(Integer, primary_key=True) - label = Column(String) - right_nodes = relationship("Node", - secondary="node_to_node", - primaryjoin="Node.id==node_to_node.c.left_node_id", - secondaryjoin="Node.id==node_to_node.c.right_node_id", - backref="left_nodes" - ) - -A classical mapping situation here is similar, where ``node_to_node`` can be joined -to ``node.c.id``:: - - from sqlalchemy import Integer, ForeignKey, String, Column, Table, MetaData - from sqlalchemy.orm import relationship, mapper - - metadata = MetaData() - - node_to_node = Table("node_to_node", metadata, - Column("left_node_id", Integer, ForeignKey("node.id"), primary_key=True), - Column("right_node_id", Integer, ForeignKey("node.id"), primary_key=True) - ) - - node = Table("node", metadata, - Column('id', Integer, primary_key=True), - Column('label', String) - ) - class Node(object): - pass - - mapper(Node, node, properties={ - 'right_nodes':relationship(Node, - secondary=node_to_node, - primaryjoin=node.c.id==node_to_node.c.left_node_id, - secondaryjoin=node.c.id==node_to_node.c.right_node_id, - backref="left_nodes" - )}) - - -Note that in both examples, the :paramref:`~.relationship.backref` -keyword specifies a ``left_nodes`` backref - when -:func:`.relationship` creates the second relationship in the reverse -direction, it's smart enough to reverse the -:paramref:`~.relationship.primaryjoin` and -:paramref:`~.relationship.secondaryjoin` arguments. - -.. _composite_secondary_join: - -Composite "Secondary" Joins -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: - - This section features some new and experimental features of SQLAlchemy. - -Sometimes, when one seeks to build a :func:`.relationship` between two tables -there is a need for more than just two or three tables to be involved in -order to join them. This is an area of :func:`.relationship` where one seeks -to push the boundaries of what's possible, and often the ultimate solution to -many of these exotic use cases needs to be hammered out on the SQLAlchemy mailing -list. - -In more recent versions of SQLAlchemy, the :paramref:`~.relationship.secondary` -parameter can be used in some of these cases in order to provide a composite -target consisting of multiple tables. Below is an example of such a -join condition (requires version 0.9.2 at least to function as is):: - - class A(Base): - __tablename__ = 'a' - - id = Column(Integer, primary_key=True) - b_id = Column(ForeignKey('b.id')) - - d = relationship("D", - secondary="join(B, D, B.d_id == D.id)." - "join(C, C.d_id == D.id)", - primaryjoin="and_(A.b_id == B.id, A.id == C.a_id)", - secondaryjoin="D.id == B.d_id", - uselist=False - ) - - class B(Base): - __tablename__ = 'b' - - id = Column(Integer, primary_key=True) - d_id = Column(ForeignKey('d.id')) - - class C(Base): - __tablename__ = 'c' - - id = Column(Integer, primary_key=True) - a_id = Column(ForeignKey('a.id')) - d_id = Column(ForeignKey('d.id')) - - class D(Base): - __tablename__ = 'd' - - id = Column(Integer, primary_key=True) - -In the above example, we provide all three of :paramref:`~.relationship.secondary`, -:paramref:`~.relationship.primaryjoin`, and :paramref:`~.relationship.secondaryjoin`, -in the declarative style referring to the named tables ``a``, ``b``, ``c``, ``d`` -directly. A query from ``A`` to ``D`` looks like: - -.. sourcecode:: python+sql - - sess.query(A).join(A.d).all() - - {opensql}SELECT a.id AS a_id, a.b_id AS a_b_id - FROM a JOIN ( - b AS b_1 JOIN d AS d_1 ON b_1.d_id = d_1.id - JOIN c AS c_1 ON c_1.d_id = d_1.id) - ON a.b_id = b_1.id AND a.id = c_1.a_id JOIN d ON d.id = b_1.d_id - -In the above example, we take advantage of being able to stuff multiple -tables into a "secondary" container, so that we can join across many -tables while still keeping things "simple" for :func:`.relationship`, in that -there's just "one" table on both the "left" and the "right" side; the -complexity is kept within the middle. - -.. versionadded:: 0.9.2 Support is improved for allowing a :func:`.join()` - construct to be used directly as the target of the :paramref:`~.relationship.secondary` - argument, including support for joins, eager joins and lazy loading, - as well as support within declarative to specify complex conditions such - as joins involving class names as targets. - -.. _relationship_non_primary_mapper: - -Relationship to Non Primary Mapper -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the previous section, we illustrated a technique where we used -:paramref:`~.relationship.secondary` in order to place additional -tables within a join condition. There is one complex join case where -even this technique is not sufficient; when we seek to join from ``A`` -to ``B``, making use of any number of ``C``, ``D``, etc. in between, -however there are also join conditions between ``A`` and ``B`` -*directly*. In this case, the join from ``A`` to ``B`` may be -difficult to express with just a complex -:paramref:`~.relationship.primaryjoin` condition, as the intermediary -tables may need special handling, and it is also not expressable with -a :paramref:`~.relationship.secondary` object, since the -``A->secondary->B`` pattern does not support any references between -``A`` and ``B`` directly. When this **extremely advanced** case -arises, we can resort to creating a second mapping as a target for the -relationship. This is where we use :func:`.mapper` in order to make a -mapping to a class that includes all the additional tables we need for -this join. In order to produce this mapper as an "alternative" mapping -for our class, we use the :paramref:`~.mapper.non_primary` flag. - -Below illustrates a :func:`.relationship` with a simple join from ``A`` to -``B``, however the primaryjoin condition is augmented with two additional -entities ``C`` and ``D``, which also must have rows that line up with -the rows in both ``A`` and ``B`` simultaneously:: - - class A(Base): - __tablename__ = 'a' - - id = Column(Integer, primary_key=True) - b_id = Column(ForeignKey('b.id')) - - class B(Base): - __tablename__ = 'b' - - id = Column(Integer, primary_key=True) - - class C(Base): - __tablename__ = 'c' - - id = Column(Integer, primary_key=True) - a_id = Column(ForeignKey('a.id')) - - class D(Base): - __tablename__ = 'd' - - id = Column(Integer, primary_key=True) - c_id = Column(ForeignKey('c.id')) - b_id = Column(ForeignKey('b.id')) - - # 1. set up the join() as a variable, so we can refer - # to it in the mapping multiple times. - j = join(B, D, D.b_id == B.id).join(C, C.id == D.c_id) - - # 2. Create a new mapper() to B, with non_primary=True. - # Columns in the join with the same name must be - # disambiguated within the mapping, using named properties. - B_viacd = mapper(B, j, non_primary=True, properties={ - "b_id": [j.c.b_id, j.c.d_b_id], - "d_id": j.c.d_id - }) - - A.b = relationship(B_viacd, primaryjoin=A.b_id == B_viacd.c.b_id) - -In the above case, our non-primary mapper for ``B`` will emit for -additional columns when we query; these can be ignored: - -.. sourcecode:: python+sql - - sess.query(A).join(A.b).all() - - {opensql}SELECT a.id AS a_id, a.b_id AS a_b_id - FROM a JOIN (b JOIN d ON d.b_id = b.id JOIN c ON c.id = d.c_id) ON a.b_id = b.id - - -Building Query-Enabled Properties -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Very ambitious custom join conditions may fail to be directly persistable, and -in some cases may not even load correctly. To remove the persistence part of -the equation, use the flag :paramref:`~.relationship.viewonly` on the -:func:`~sqlalchemy.orm.relationship`, which establishes it as a read-only -attribute (data written to the collection will be ignored on flush()). -However, in extreme cases, consider using a regular Python property in -conjunction with :class:`.Query` as follows: - -.. sourcecode:: python+sql - - class User(Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - - def _get_addresses(self): - return object_session(self).query(Address).with_parent(self).filter(...).all() - addresses = property(_get_addresses) - - -.. _post_update: - -Rows that point to themselves / Mutually Dependent Rows -------------------------------------------------------- - -This is a very specific case where relationship() must perform an INSERT and a -second UPDATE in order to properly populate a row (and vice versa an UPDATE -and DELETE in order to delete without violating foreign key constraints). The -two use cases are: - -* A table contains a foreign key to itself, and a single row will - have a foreign key value pointing to its own primary key. -* Two tables each contain a foreign key referencing the other - table, with a row in each table referencing the other. - -For example:: - - user - --------------------------------- - user_id name related_user_id - 1 'ed' 1 - -Or:: - - widget entry - ------------------------------------------- --------------------------------- - widget_id name favorite_entry_id entry_id name widget_id - 1 'somewidget' 5 5 'someentry' 1 - -In the first case, a row points to itself. Technically, a database that uses -sequences such as PostgreSQL or Oracle can INSERT the row at once using a -previously generated value, but databases which rely upon autoincrement-style -primary key identifiers cannot. The :func:`~sqlalchemy.orm.relationship` -always assumes a "parent/child" model of row population during flush, so -unless you are populating the primary key/foreign key columns directly, -:func:`~sqlalchemy.orm.relationship` needs to use two statements. - -In the second case, the "widget" row must be inserted before any referring -"entry" rows, but then the "favorite_entry_id" column of that "widget" row -cannot be set until the "entry" rows have been generated. In this case, it's -typically impossible to insert the "widget" and "entry" rows using just two -INSERT statements; an UPDATE must be performed in order to keep foreign key -constraints fulfilled. The exception is if the foreign keys are configured as -"deferred until commit" (a feature some databases support) and if the -identifiers were populated manually (again essentially bypassing -:func:`~sqlalchemy.orm.relationship`). - -To enable the usage of a supplementary UPDATE statement, -we use the :paramref:`~.relationship.post_update` option -of :func:`.relationship`. This specifies that the linkage between the -two rows should be created using an UPDATE statement after both rows -have been INSERTED; it also causes the rows to be de-associated with -each other via UPDATE before a DELETE is emitted. The flag should -be placed on just *one* of the relationships, preferably the -many-to-one side. Below we illustrate -a complete example, including two :class:`.ForeignKey` constructs, one which -specifies :paramref:`~.ForeignKey.use_alter` to help with emitting CREATE TABLE statements:: - - from sqlalchemy import Integer, ForeignKey, Column - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class Entry(Base): - __tablename__ = 'entry' - entry_id = Column(Integer, primary_key=True) - widget_id = Column(Integer, ForeignKey('widget.widget_id')) - name = Column(String(50)) - - class Widget(Base): - __tablename__ = 'widget' - - widget_id = Column(Integer, primary_key=True) - favorite_entry_id = Column(Integer, - ForeignKey('entry.entry_id', - use_alter=True, - name="fk_favorite_entry")) - name = Column(String(50)) - - entries = relationship(Entry, primaryjoin= - widget_id==Entry.widget_id) - favorite_entry = relationship(Entry, - primaryjoin= - favorite_entry_id==Entry.entry_id, - post_update=True) - -When a structure against the above configuration is flushed, the "widget" row will be -INSERTed minus the "favorite_entry_id" value, then all the "entry" rows will -be INSERTed referencing the parent "widget" row, and then an UPDATE statement -will populate the "favorite_entry_id" column of the "widget" table (it's one -row at a time for the time being): - -.. sourcecode:: pycon+sql - - >>> w1 = Widget(name='somewidget') - >>> e1 = Entry(name='someentry') - >>> w1.favorite_entry = e1 - >>> w1.entries = [e1] - >>> session.add_all([w1, e1]) - {sql}>>> session.commit() - BEGIN (implicit) - INSERT INTO widget (favorite_entry_id, name) VALUES (?, ?) - (None, 'somewidget') - INSERT INTO entry (widget_id, name) VALUES (?, ?) - (1, 'someentry') - UPDATE widget SET favorite_entry_id=? WHERE widget.widget_id = ? - (1, 1) - COMMIT - -An additional configuration we can specify is to supply a more -comprehensive foreign key constraint on ``Widget``, such that -it's guaranteed that ``favorite_entry_id`` refers to an ``Entry`` -that also refers to this ``Widget``. We can use a composite foreign key, -as illustrated below:: - - from sqlalchemy import Integer, ForeignKey, String, \ - Column, UniqueConstraint, ForeignKeyConstraint - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - Base = declarative_base() - - class Entry(Base): - __tablename__ = 'entry' - entry_id = Column(Integer, primary_key=True) - widget_id = Column(Integer, ForeignKey('widget.widget_id')) - name = Column(String(50)) - __table_args__ = ( - UniqueConstraint("entry_id", "widget_id"), - ) - - class Widget(Base): - __tablename__ = 'widget' - - widget_id = Column(Integer, autoincrement='ignore_fk', primary_key=True) - favorite_entry_id = Column(Integer) - - name = Column(String(50)) - - __table_args__ = ( - ForeignKeyConstraint( - ["widget_id", "favorite_entry_id"], - ["entry.widget_id", "entry.entry_id"], - name="fk_favorite_entry", use_alter=True - ), - ) - - entries = relationship(Entry, primaryjoin= - widget_id==Entry.widget_id, - foreign_keys=Entry.widget_id) - favorite_entry = relationship(Entry, - primaryjoin= - favorite_entry_id==Entry.entry_id, - foreign_keys=favorite_entry_id, - post_update=True) - -The above mapping features a composite :class:`.ForeignKeyConstraint` -bridging the ``widget_id`` and ``favorite_entry_id`` columns. To ensure -that ``Widget.widget_id`` remains an "autoincrementing" column we specify -:paramref:`~.Column.autoincrement` to the value ``"ignore_fk"`` -on :class:`.Column`, and additionally on each -:func:`.relationship` we must limit those columns considered as part of -the foreign key for the purposes of joining and cross-population. - -.. _passive_updates: - -Mutable Primary Keys / Update Cascades ---------------------------------------- - -When the primary key of an entity changes, related items -which reference the primary key must also be updated as -well. For databases which enforce referential integrity, -it's required to use the database's ON UPDATE CASCADE -functionality in order to propagate primary key changes -to referenced foreign keys - the values cannot be out -of sync for any moment. - -For databases that don't support this, such as SQLite and -MySQL without their referential integrity options turned -on, the :paramref:`~.relationship.passive_updates` flag can -be set to ``False``, most preferably on a one-to-many or -many-to-many :func:`.relationship`, which instructs -SQLAlchemy to issue UPDATE statements individually for -objects referenced in the collection, loading them into -memory if not already locally present. The -:paramref:`~.relationship.passive_updates` flag can also be ``False`` in -conjunction with ON UPDATE CASCADE functionality, -although in that case the unit of work will be issuing -extra SELECT and UPDATE statements unnecessarily. - -A typical mutable primary key setup might look like:: - - class User(Base): - __tablename__ = 'user' - - username = Column(String(50), primary_key=True) - fullname = Column(String(100)) - - # passive_updates=False *only* needed if the database - # does not implement ON UPDATE CASCADE - addresses = relationship("Address", passive_updates=False) - - class Address(Base): - __tablename__ = 'address' - - email = Column(String(50), primary_key=True) - username = Column(String(50), - ForeignKey('user.username', onupdate="cascade") - ) - -:paramref:`~.relationship.passive_updates` is set to ``True`` by default, -indicating that ON UPDATE CASCADE is expected to be in -place in the usual case for foreign keys that expect -to have a mutating parent key. - -A :paramref:`~.relationship.passive_updates` setting of False may be configured on any -direction of relationship, i.e. one-to-many, many-to-one, -and many-to-many, although it is much more effective when -placed just on the one-to-many or many-to-many side. -Configuring the :paramref:`~.relationship.passive_updates` -to False only on the -many-to-one side will have only a partial effect, as the -unit of work searches only through the current identity -map for objects that may be referencing the one with a -mutating primary key, not throughout the database. - -Relationships API ------------------ - -.. autofunction:: relationship - -.. autofunction:: backref - -.. autofunction:: relation - -.. autofunction:: dynamic_loader - -.. autofunction:: foreign - -.. autofunction:: remote - - +.. toctree:: + :maxdepth: 2 + + basic_relationships + self_referential + backref + join_conditions + collections + relationship_persistence + relationship_api diff --git a/doc/build/orm/scalar_mapping.rst b/doc/build/orm/scalar_mapping.rst new file mode 100644 index 000000000..65efd5dbd --- /dev/null +++ b/doc/build/orm/scalar_mapping.rst @@ -0,0 +1,18 @@ +.. module:: sqlalchemy.orm + +=============================== +Mapping Columns and Expressions +=============================== + +The following sections discuss how table columns and SQL expressions are +mapped to individual object attributes. + +.. toctree:: + :maxdepth: 2 + + mapping_columns + mapped_sql_expr + mapped_attributes + composites + + diff --git a/doc/build/orm/self_referential.rst b/doc/build/orm/self_referential.rst new file mode 100644 index 000000000..f6ed35fd6 --- /dev/null +++ b/doc/build/orm/self_referential.rst @@ -0,0 +1,261 @@ +.. _self_referential: + +Adjacency List Relationships +----------------------------- + +The **adjacency list** pattern is a common relational pattern whereby a table +contains a foreign key reference to itself. This is the most common +way to represent hierarchical data in flat tables. Other methods +include **nested sets**, sometimes called "modified preorder", +as well as **materialized path**. Despite the appeal that modified preorder +has when evaluated for its fluency within SQL queries, the adjacency list model is +probably the most appropriate pattern for the large majority of hierarchical +storage needs, for reasons of concurrency, reduced complexity, and that +modified preorder has little advantage over an application which can fully +load subtrees into the application space. + +In this example, we'll work with a single mapped +class called ``Node``, representing a tree structure:: + + class Node(Base): + __tablename__ = 'node' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('node.id')) + data = Column(String(50)) + children = relationship("Node") + +With this structure, a graph such as the following:: + + root --+---> child1 + +---> child2 --+--> subchild1 + | +--> subchild2 + +---> child3 + +Would be represented with data such as:: + + id parent_id data + --- ------- ---- + 1 NULL root + 2 1 child1 + 3 1 child2 + 4 3 subchild1 + 5 3 subchild2 + 6 1 child3 + +The :func:`.relationship` configuration here works in the +same way as a "normal" one-to-many relationship, with the +exception that the "direction", i.e. whether the relationship +is one-to-many or many-to-one, is assumed by default to +be one-to-many. To establish the relationship as many-to-one, +an extra directive is added known as :paramref:`~.relationship.remote_side`, which +is a :class:`.Column` or collection of :class:`.Column` objects +that indicate those which should be considered to be "remote":: + + class Node(Base): + __tablename__ = 'node' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('node.id')) + data = Column(String(50)) + parent = relationship("Node", remote_side=[id]) + +Where above, the ``id`` column is applied as the :paramref:`~.relationship.remote_side` +of the ``parent`` :func:`.relationship`, thus establishing +``parent_id`` as the "local" side, and the relationship +then behaves as a many-to-one. + +As always, both directions can be combined into a bidirectional +relationship using the :func:`.backref` function:: + + class Node(Base): + __tablename__ = 'node' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('node.id')) + data = Column(String(50)) + children = relationship("Node", + backref=backref('parent', remote_side=[id]) + ) + +There are several examples included with SQLAlchemy illustrating +self-referential strategies; these include :ref:`examples_adjacencylist` and +:ref:`examples_xmlpersistence`. + +Composite Adjacency Lists +~~~~~~~~~~~~~~~~~~~~~~~~~ + +A sub-category of the adjacency list relationship is the rare +case where a particular column is present on both the "local" and +"remote" side of the join condition. An example is the ``Folder`` +class below; using a composite primary key, the ``account_id`` +column refers to itself, to indicate sub folders which are within +the same account as that of the parent; while ``folder_id`` refers +to a specific folder within that account:: + + class Folder(Base): + __tablename__ = 'folder' + __table_args__ = ( + ForeignKeyConstraint( + ['account_id', 'parent_id'], + ['folder.account_id', 'folder.folder_id']), + ) + + account_id = Column(Integer, primary_key=True) + folder_id = Column(Integer, primary_key=True) + parent_id = Column(Integer) + name = Column(String) + + parent_folder = relationship("Folder", + backref="child_folders", + remote_side=[account_id, folder_id] + ) + +Above, we pass ``account_id`` into the :paramref:`~.relationship.remote_side` list. +:func:`.relationship` recognizes that the ``account_id`` column here +is on both sides, and aligns the "remote" column along with the +``folder_id`` column, which it recognizes as uniquely present on +the "remote" side. + +.. versionadded:: 0.8 + Support for self-referential composite keys in :func:`.relationship` + where a column points to itself. + +Self-Referential Query Strategies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Querying of self-referential structures works like any other query:: + + # get all nodes named 'child2' + session.query(Node).filter(Node.data=='child2') + +However extra care is needed when attempting to join along +the foreign key from one level of the tree to the next. In SQL, +a join from a table to itself requires that at least one side of the +expression be "aliased" so that it can be unambiguously referred to. + +Recall from :ref:`ormtutorial_aliases` in the ORM tutorial that the +:func:`.orm.aliased` construct is normally used to provide an "alias" of +an ORM entity. Joining from ``Node`` to itself using this technique +looks like: + +.. sourcecode:: python+sql + + from sqlalchemy.orm import aliased + + nodealias = aliased(Node) + {sql}session.query(Node).filter(Node.data=='subchild1').\ + join(nodealias, Node.parent).\ + filter(nodealias.data=="child2").\ + all() + SELECT node.id AS node_id, + node.parent_id AS node_parent_id, + node.data AS node_data + FROM node JOIN node AS node_1 + ON node.parent_id = node_1.id + WHERE node.data = ? + AND node_1.data = ? + ['subchild1', 'child2'] + +:meth:`.Query.join` also includes a feature known as +:paramref:`.Query.join.aliased` that can shorten the verbosity self- +referential joins, at the expense of query flexibility. This feature +performs a similar "aliasing" step to that above, without the need for +an explicit entity. Calls to :meth:`.Query.filter` and similar +subsequent to the aliased join will **adapt** the ``Node`` entity to +be that of the alias: + +.. sourcecode:: python+sql + + {sql}session.query(Node).filter(Node.data=='subchild1').\ + join(Node.parent, aliased=True).\ + filter(Node.data=='child2').\ + all() + SELECT node.id AS node_id, + node.parent_id AS node_parent_id, + node.data AS node_data + FROM node + JOIN node AS node_1 ON node_1.id = node.parent_id + WHERE node.data = ? AND node_1.data = ? + ['subchild1', 'child2'] + +To add criterion to multiple points along a longer join, add +:paramref:`.Query.join.from_joinpoint` to the additional +:meth:`~.Query.join` calls: + +.. sourcecode:: python+sql + + # get all nodes named 'subchild1' with a + # parent named 'child2' and a grandparent 'root' + {sql}session.query(Node).\ + filter(Node.data=='subchild1').\ + join(Node.parent, aliased=True).\ + filter(Node.data=='child2').\ + join(Node.parent, aliased=True, from_joinpoint=True).\ + filter(Node.data=='root').\ + all() + SELECT node.id AS node_id, + node.parent_id AS node_parent_id, + node.data AS node_data + FROM node + JOIN node AS node_1 ON node_1.id = node.parent_id + JOIN node AS node_2 ON node_2.id = node_1.parent_id + WHERE node.data = ? + AND node_1.data = ? + AND node_2.data = ? + ['subchild1', 'child2', 'root'] + +:meth:`.Query.reset_joinpoint` will also remove the "aliasing" from filtering +calls:: + + session.query(Node).\ + join(Node.children, aliased=True).\ + filter(Node.data == 'foo').\ + reset_joinpoint().\ + filter(Node.data == 'bar') + +For an example of using :paramref:`.Query.join.aliased` to +arbitrarily join along a chain of self-referential nodes, see +:ref:`examples_xmlpersistence`. + +.. _self_referential_eager_loading: + +Configuring Self-Referential Eager Loading +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Eager loading of relationships occurs using joins or outerjoins from parent to +child table during a normal query operation, such that the parent and its +immediate child collection or reference can be populated from a single SQL +statement, or a second statement for all immediate child collections. +SQLAlchemy's joined and subquery eager loading use aliased tables in all cases +when joining to related items, so are compatible with self-referential +joining. However, to use eager loading with a self-referential relationship, +SQLAlchemy needs to be told how many levels deep it should join and/or query; +otherwise the eager load will not take place at all. This depth setting is +configured via :paramref:`~.relationships.join_depth`: + +.. sourcecode:: python+sql + + class Node(Base): + __tablename__ = 'node' + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('node.id')) + data = Column(String(50)) + children = relationship("Node", + lazy="joined", + join_depth=2) + + {sql}session.query(Node).all() + SELECT node_1.id AS node_1_id, + node_1.parent_id AS node_1_parent_id, + node_1.data AS node_1_data, + node_2.id AS node_2_id, + node_2.parent_id AS node_2_parent_id, + node_2.data AS node_2_data, + node.id AS node_id, + node.parent_id AS node_parent_id, + node.data AS node_data + FROM node + LEFT OUTER JOIN node AS node_2 + ON node.id = node_2.parent_id + LEFT OUTER JOIN node AS node_1 + ON node_2.id = node_1.parent_id + [] + diff --git a/doc/build/orm/session.rst b/doc/build/orm/session.rst index 97506e210..624ee9f75 100644 --- a/doc/build/orm/session.rst +++ b/doc/build/orm/session.rst @@ -11,2663 +11,15 @@ are the primary configurational interface for the ORM. Once mappings are configured, the primary usage interface for persistence operations is the :class:`.Session`. -What does the Session do ? -========================== +.. toctree:: + :maxdepth: 2 -In the most general sense, the :class:`~.Session` establishes all -conversations with the database and represents a "holding zone" for all the -objects which you've loaded or associated with it during its lifespan. It -provides the entrypoint to acquire a :class:`.Query` object, which sends -queries to the database using the :class:`~.Session` object's current database -connection, populating result rows into objects that are then stored in the -:class:`.Session`, inside a structure called the `Identity Map -`_ - a data structure -that maintains unique copies of each object, where "unique" means "only one -object with a particular primary key". + session_basics + session_state_management + cascades + session_transaction + persistence_techniques + contextual + session_api -The :class:`.Session` begins in an essentially stateless form. Once queries -are issued or other objects are persisted with it, it requests a connection -resource from an :class:`.Engine` that is associated either with the -:class:`.Session` itself or with the mapped :class:`.Table` objects being -operated upon. This connection represents an ongoing transaction, which -remains in effect until the :class:`.Session` is instructed to commit or roll -back its pending state. - -All changes to objects maintained by a :class:`.Session` are tracked - before -the database is queried again or before the current transaction is committed, -it **flushes** all pending changes to the database. This is known as the `Unit -of Work `_ pattern. - -When using a :class:`.Session`, it's important to note that the objects -which are associated with it are **proxy objects** to the transaction being -held by the :class:`.Session` - there are a variety of events that will cause -objects to re-access the database in order to keep synchronized. It is -possible to "detach" objects from a :class:`.Session`, and to continue using -them, though this practice has its caveats. It's intended that -usually, you'd re-associate detached objects with another :class:`.Session` when you -want to work with them again, so that they can resume their normal task of -representing database state. - -.. _session_getting: - -Getting a Session -================= - -:class:`.Session` is a regular Python class which can -be directly instantiated. However, to standardize how sessions are configured -and acquired, the :class:`.sessionmaker` class is normally -used to create a top level :class:`.Session` -configuration which can then be used throughout an application without the -need to repeat the configurational arguments. - -The usage of :class:`.sessionmaker` is illustrated below: - -.. sourcecode:: python+sql - - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - - # an Engine, which the Session will use for connection - # resources - some_engine = create_engine('postgresql://scott:tiger@localhost/') - - # create a configured "Session" class - Session = sessionmaker(bind=some_engine) - - # create a Session - session = Session() - - # work with sess - myobject = MyObject('foo', 'bar') - session.add(myobject) - session.commit() - -Above, the :class:`.sessionmaker` call creates a factory for us, -which we assign to the name ``Session``. This factory, when -called, will create a new :class:`.Session` object using the configurational -arguments we've given the factory. In this case, as is typical, -we've configured the factory to specify a particular :class:`.Engine` for -connection resources. - -A typical setup will associate the :class:`.sessionmaker` with an :class:`.Engine`, -so that each :class:`.Session` generated will use this :class:`.Engine` -to acquire connection resources. This association can -be set up as in the example above, using the ``bind`` argument. - -When you write your application, place the -:class:`.sessionmaker` factory at the global level. This -factory can then -be used by the rest of the applcation as the source of new :class:`.Session` -instances, keeping the configuration for how :class:`.Session` objects -are constructed in one place. - -The :class:`.sessionmaker` factory can also be used in conjunction with -other helpers, which are passed a user-defined :class:`.sessionmaker` that -is then maintained by the helper. Some of these helpers are discussed in the -section :ref:`session_faq_whentocreate`. - -Adding Additional Configuration to an Existing sessionmaker() --------------------------------------------------------------- - -A common scenario is where the :class:`.sessionmaker` is invoked -at module import time, however the generation of one or more :class:`.Engine` -instances to be associated with the :class:`.sessionmaker` has not yet proceeded. -For this use case, the :class:`.sessionmaker` construct offers the -:meth:`.sessionmaker.configure` method, which will place additional configuration -directives into an existing :class:`.sessionmaker` that will take place -when the construct is invoked:: - - - from sqlalchemy.orm import sessionmaker - from sqlalchemy import create_engine - - # configure Session class with desired options - Session = sessionmaker() - - # later, we create the engine - engine = create_engine('postgresql://...') - - # associate it with our custom Session class - Session.configure(bind=engine) - - # work with the session - session = Session() - -Creating Ad-Hoc Session Objects with Alternate Arguments ---------------------------------------------------------- - -For the use case where an application needs to create a new :class:`.Session` with -special arguments that deviate from what is normally used throughout the application, -such as a :class:`.Session` that binds to an alternate -source of connectivity, or a :class:`.Session` that should -have other arguments such as ``expire_on_commit`` established differently from -what most of the application wants, specific arguments can be passed to the -:class:`.sessionmaker` factory's :meth:`.sessionmaker.__call__` method. -These arguments will override whatever -configurations have already been placed, such as below, where a new :class:`.Session` -is constructed against a specific :class:`.Connection`:: - - # at the module level, the global sessionmaker, - # bound to a specific Engine - Session = sessionmaker(bind=engine) - - # later, some unit of code wants to create a - # Session that is bound to a specific Connection - conn = engine.connect() - session = Session(bind=conn) - -The typical rationale for the association of a :class:`.Session` with a specific -:class:`.Connection` is that of a test fixture that maintains an external -transaction - see :ref:`session_external_transaction` for an example of this. - -Using the Session -================== - -.. _session_object_states: - -Quickie Intro to Object States ------------------------------- - -It's helpful to know the states which an instance can have within a session: - -* **Transient** - an instance that's not in a session, and is not saved to the - database; i.e. it has no database identity. The only relationship such an - object has to the ORM is that its class has a ``mapper()`` associated with - it. - -* **Pending** - when you :meth:`~.Session.add` a transient - instance, it becomes pending. It still wasn't actually flushed to the - database yet, but it will be when the next flush occurs. - -* **Persistent** - An instance which is present in the session and has a record - in the database. You get persistent instances by either flushing so that the - pending instances become persistent, or by querying the database for - existing instances (or moving persistent instances from other sessions into - your local session). - -* **Detached** - an instance which has a record in the database, but is not in - any session. There's nothing wrong with this, and you can use objects - normally when they're detached, **except** they will not be able to issue - any SQL in order to load collections or attributes which are not yet loaded, - or were marked as "expired". - -Knowing these states is important, since the -:class:`.Session` tries to be strict about ambiguous -operations (such as trying to save the same object to two different sessions -at the same time). - -Getting the Current State of an Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The actual state of any mapped object can be viewed at any time using -the :func:`.inspect` system:: - - >>> from sqlalchemy import inspect - >>> insp = inspect(my_object) - >>> insp.persistent - True - -.. seealso:: - - :attr:`.InstanceState.transient` - - :attr:`.InstanceState.pending` - - :attr:`.InstanceState.persistent` - - :attr:`.InstanceState.detached` - - -.. _session_faq: - -Session Frequently Asked Questions ------------------------------------ - - -When do I make a :class:`.sessionmaker`? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Just one time, somewhere in your application's global scope. It should be -looked upon as part of your application's configuration. If your -application has three .py files in a package, you could, for example, -place the :class:`.sessionmaker` line in your ``__init__.py`` file; from -that point on your other modules say "from mypackage import Session". That -way, everyone else just uses :class:`.Session()`, -and the configuration of that session is controlled by that central point. - -If your application starts up, does imports, but does not know what -database it's going to be connecting to, you can bind the -:class:`.Session` at the "class" level to the -engine later on, using :meth:`.sessionmaker.configure`. - -In the examples in this section, we will frequently show the -:class:`.sessionmaker` being created right above the line where we actually -invoke :class:`.Session`. But that's just for -example's sake! In reality, the :class:`.sessionmaker` would be somewhere -at the module level. The calls to instantiate :class:`.Session` -would then be placed at the point in the application where database -conversations begin. - -.. _session_faq_whentocreate: - -When do I construct a :class:`.Session`, when do I commit it, and when do I close it? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. topic:: tl;dr; - - As a general rule, keep the lifecycle of the session **separate and - external** from functions and objects that access and/or manipulate - database data. - -A :class:`.Session` is typically constructed at the beginning of a logical -operation where database access is potentially anticipated. - -The :class:`.Session`, whenever it is used to talk to the database, -begins a database transaction as soon as it starts communicating. -Assuming the ``autocommit`` flag is left at its recommended default -of ``False``, this transaction remains in progress until the :class:`.Session` -is rolled back, committed, or closed. The :class:`.Session` will -begin a new transaction if it is used again, subsequent to the previous -transaction ending; from this it follows that the :class:`.Session` -is capable of having a lifespan across many transactions, though only -one at a time. We refer to these two concepts as **transaction scope** -and **session scope**. - -The implication here is that the SQLAlchemy ORM is encouraging the -developer to establish these two scopes in their application, -including not only when the scopes begin and end, but also the -expanse of those scopes, for example should a single -:class:`.Session` instance be local to the execution flow within a -function or method, should it be a global object used by the -entire application, or somewhere in between these two. - -The burden placed on the developer to determine this scope is one -area where the SQLAlchemy ORM necessarily has a strong opinion -about how the database should be used. The :term:`unit of work` pattern -is specifically one of accumulating changes over time and flushing -them periodically, keeping in-memory state in sync with what's -known to be present in a local transaction. This pattern is only -effective when meaningful transaction scopes are in place. - -It's usually not very hard to determine the best points at which -to begin and end the scope of a :class:`.Session`, though the wide -variety of application architectures possible can introduce -challenging situations. - -A common choice is to tear down the :class:`.Session` at the same -time the transaction ends, meaning the transaction and session scopes -are the same. This is a great choice to start out with as it -removes the need to consider session scope as separate from transaction -scope. - -While there's no one-size-fits-all recommendation for how transaction -scope should be determined, there are common patterns. Especially -if one is writing a web application, the choice is pretty much established. - -A web application is the easiest case because such an appication is already -constructed around a single, consistent scope - this is the **request**, -which represents an incoming request from a browser, the processing -of that request to formulate a response, and finally the delivery of that -response back to the client. Integrating web applications with the -:class:`.Session` is then the straightforward task of linking the -scope of the :class:`.Session` to that of the request. The :class:`.Session` -can be established as the request begins, or using a :term:`lazy initialization` -pattern which establishes one as soon as it is needed. The request -then proceeds, with some system in place where application logic can access -the current :class:`.Session` in a manner associated with how the actual -request object is accessed. As the request ends, the :class:`.Session` -is torn down as well, usually through the usage of event hooks provided -by the web framework. The transaction used by the :class:`.Session` -may also be committed at this point, or alternatively the application may -opt for an explicit commit pattern, only committing for those requests -where one is warranted, but still always tearing down the :class:`.Session` -unconditionally at the end. - -Some web frameworks include infrastructure to assist in the task -of aligning the lifespan of a :class:`.Session` with that of a web request. -This includes products such as `Flask-SQLAlchemy `_, -for usage in conjunction with the Flask web framework, -and `Zope-SQLAlchemy `_, -typically used with the Pyramid framework. -SQLAlchemy recommends that these products be used as available. - -In those situations where the integration libraries are not -provided or are insufficient, SQLAlchemy includes its own "helper" class known as -:class:`.scoped_session`. A tutorial on the usage of this object -is at :ref:`unitofwork_contextual`. It provides both a quick way -to associate a :class:`.Session` with the current thread, as well as -patterns to associate :class:`.Session` objects with other kinds of -scopes. - -As mentioned before, for non-web applications there is no one clear -pattern, as applications themselves don't have just one pattern -of architecture. The best strategy is to attempt to demarcate -"operations", points at which a particular thread begins to perform -a series of operations for some period of time, which can be committed -at the end. Some examples: - -* A background daemon which spawns off child forks - would want to create a :class:`.Session` local to each child - process, work with that :class:`.Session` through the life of the "job" - that the fork is handling, then tear it down when the job is completed. - -* For a command-line script, the application would create a single, global - :class:`.Session` that is established when the program begins to do its - work, and commits it right as the program is completing its task. - -* For a GUI interface-driven application, the scope of the :class:`.Session` - may best be within the scope of a user-generated event, such as a button - push. Or, the scope may correspond to explicit user interaction, such as - the user "opening" a series of records, then "saving" them. - -As a general rule, the application should manage the lifecycle of the -session *externally* to functions that deal with specific data. This is a -fundamental separation of concerns which keeps data-specific operations -agnostic of the context in which they access and manipulate that data. - -E.g. **don't do this**:: - - ### this is the **wrong way to do it** ### - - class ThingOne(object): - def go(self): - session = Session() - try: - session.query(FooBar).update({"x": 5}) - session.commit() - except: - session.rollback() - raise - - class ThingTwo(object): - def go(self): - session = Session() - try: - session.query(Widget).update({"q": 18}) - session.commit() - except: - session.rollback() - raise - - def run_my_program(): - ThingOne().go() - ThingTwo().go() - -Keep the lifecycle of the session (and usually the transaction) -**separate and external**:: - - ### this is a **better** (but not the only) way to do it ### - - class ThingOne(object): - def go(self, session): - session.query(FooBar).update({"x": 5}) - - class ThingTwo(object): - def go(self, session): - session.query(Widget).update({"q": 18}) - - def run_my_program(): - session = Session() - try: - ThingOne().go(session) - ThingTwo().go(session) - - session.commit() - except: - session.rollback() - raise - finally: - session.close() - -The advanced developer will try to keep the details of session, transaction -and exception management as far as possible from the details of the program -doing its work. For example, we can further separate concerns using a `context manager `_:: - - ### another way (but again *not the only way*) to do it ### - - from contextlib import contextmanager - - @contextmanager - def session_scope(): - """Provide a transactional scope around a series of operations.""" - session = Session() - try: - yield session - session.commit() - except: - session.rollback() - raise - finally: - session.close() - - - def run_my_program(): - with session_scope() as session: - ThingOne().go(session) - ThingTwo().go(session) - - -Is the Session a cache? -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Yeee...no. It's somewhat used as a cache, in that it implements the -:term:`identity map` pattern, and stores objects keyed to their primary key. -However, it doesn't do any kind of query caching. This means, if you say -``session.query(Foo).filter_by(name='bar')``, even if ``Foo(name='bar')`` -is right there, in the identity map, the session has no idea about that. -It has to issue SQL to the database, get the rows back, and then when it -sees the primary key in the row, *then* it can look in the local identity -map and see that the object is already there. It's only when you say -``query.get({some primary key})`` that the -:class:`~sqlalchemy.orm.session.Session` doesn't have to issue a query. - -Additionally, the Session stores object instances using a weak reference -by default. This also defeats the purpose of using the Session as a cache. - -The :class:`.Session` is not designed to be a -global object from which everyone consults as a "registry" of objects. -That's more the job of a **second level cache**. SQLAlchemy provides -a pattern for implementing second level caching using `dogpile.cache `_, -via the :ref:`examples_caching` example. - -How can I get the :class:`~sqlalchemy.orm.session.Session` for a certain object? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Use the :meth:`~.Session.object_session` classmethod -available on :class:`~sqlalchemy.orm.session.Session`:: - - session = Session.object_session(someobject) - -The newer :ref:`core_inspection_toplevel` system can also be used:: - - from sqlalchemy import inspect - session = inspect(someobject).session - -.. _session_faq_threadsafe: - -Is the session thread-safe? -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`.Session` is very much intended to be used in a -**non-concurrent** fashion, which usually means in only one thread at a -time. - -The :class:`.Session` should be used in such a way that one -instance exists for a single series of operations within a single -transaction. One expedient way to get this effect is by associating -a :class:`.Session` with the current thread (see :ref:`unitofwork_contextual` -for background). Another is to use a pattern -where the :class:`.Session` is passed between functions and is otherwise -not shared with other threads. - -The bigger point is that you should not *want* to use the session -with multiple concurrent threads. That would be like having everyone at a -restaurant all eat from the same plate. The session is a local "workspace" -that you use for a specific set of tasks; you don't want to, or need to, -share that session with other threads who are doing some other task. - -Making sure the :class:`.Session` is only used in a single concurrent thread at a time -is called a "share nothing" approach to concurrency. But actually, not -sharing the :class:`.Session` implies a more significant pattern; it -means not just the :class:`.Session` object itself, but -also **all objects that are associated with that Session**, must be kept within -the scope of a single concurrent thread. The set of mapped -objects associated with a :class:`.Session` are essentially proxies for data -within database rows accessed over a database connection, and so just like -the :class:`.Session` itself, the whole -set of objects is really just a large-scale proxy for a database connection -(or connections). Ultimately, it's mostly the DBAPI connection itself that -we're keeping away from concurrent access; but since the :class:`.Session` -and all the objects associated with it are all proxies for that DBAPI connection, -the entire graph is essentially not safe for concurrent access. - -If there are in fact multiple threads participating -in the same task, then you may consider sharing the session and its objects between -those threads; however, in this extremely unusual scenario the application would -need to ensure that a proper locking scheme is implemented so that there isn't -*concurrent* access to the :class:`.Session` or its state. A more common approach -to this situation is to maintain a single :class:`.Session` per concurrent thread, -but to instead *copy* objects from one :class:`.Session` to another, often -using the :meth:`.Session.merge` method to copy the state of an object into -a new object local to a different :class:`.Session`. - -Querying --------- - -The :meth:`~.Session.query` function takes one or more -*entities* and returns a new :class:`~sqlalchemy.orm.query.Query` object which -will issue mapper queries within the context of this Session. An entity is -defined as a mapped class, a :class:`~sqlalchemy.orm.mapper.Mapper` object, an -orm-enabled *descriptor*, or an ``AliasedClass`` object:: - - # query from a class - session.query(User).filter_by(name='ed').all() - - # query with multiple classes, returns tuples - session.query(User, Address).join('addresses').filter_by(name='ed').all() - - # query using orm-enabled descriptors - session.query(User.name, User.fullname).all() - - # query from a mapper - user_mapper = class_mapper(User) - session.query(user_mapper) - -When :class:`~sqlalchemy.orm.query.Query` returns results, each object -instantiated is stored within the identity map. When a row matches an object -which is already present, the same object is returned. In the latter case, -whether or not the row is populated onto an existing object depends upon -whether the attributes of the instance have been *expired* or not. A -default-configured :class:`~sqlalchemy.orm.session.Session` automatically -expires all instances along transaction boundaries, so that with a normally -isolated transaction, there shouldn't be any issue of instances representing -data which is stale with regards to the current transaction. - -The :class:`.Query` object is introduced in great detail in -:ref:`ormtutorial_toplevel`, and further documented in -:ref:`query_api_toplevel`. - -Adding New or Existing Items ----------------------------- - -:meth:`~.Session.add` is used to place instances in the -session. For *transient* (i.e. brand new) instances, this will have the effect -of an INSERT taking place for those instances upon the next flush. For -instances which are *persistent* (i.e. were loaded by this session), they are -already present and do not need to be added. Instances which are *detached* -(i.e. have been removed from a session) may be re-associated with a session -using this method:: - - user1 = User(name='user1') - user2 = User(name='user2') - session.add(user1) - session.add(user2) - - session.commit() # write changes to the database - -To add a list of items to the session at once, use -:meth:`~.Session.add_all`:: - - session.add_all([item1, item2, item3]) - -The :meth:`~.Session.add` operation **cascades** along -the ``save-update`` cascade. For more details see the section -:ref:`unitofwork_cascades`. - -.. _unitofwork_merging: - -Merging -------- - -:meth:`~.Session.merge` transfers state from an -outside object into a new or already existing instance within a session. It -also reconciles the incoming data against the state of the -database, producing a history stream which will be applied towards the next -flush, or alternatively can be made to produce a simple "transfer" of -state without producing change history or accessing the database. Usage is as follows:: - - merged_object = session.merge(existing_object) - -When given an instance, it follows these steps: - -* It examines the primary key of the instance. If it's present, it attempts - to locate that instance in the local identity map. If the ``load=True`` - flag is left at its default, it also checks the database for this primary - key if not located locally. -* If the given instance has no primary key, or if no instance can be found - with the primary key given, a new instance is created. -* The state of the given instance is then copied onto the located/newly - created instance. For attributes which are present on the source - instance, the value is transferred to the target instance. For mapped - attributes which aren't present on the source, the attribute is - expired on the target instance, discarding its existing value. - - If the ``load=True`` flag is left at its default, - this copy process emits events and will load the target object's - unloaded collections for each attribute present on the source object, - so that the incoming state can be reconciled against what's - present in the database. If ``load`` - is passed as ``False``, the incoming data is "stamped" directly without - producing any history. -* The operation is cascaded to related objects and collections, as - indicated by the ``merge`` cascade (see :ref:`unitofwork_cascades`). -* The new instance is returned. - -With :meth:`~.Session.merge`, the given "source" -instance is not modified nor is it associated with the target :class:`.Session`, -and remains available to be merged with any number of other :class:`.Session` -objects. :meth:`~.Session.merge` is useful for -taking the state of any kind of object structure without regard for its -origins or current session associations and copying its state into a -new session. Here's some examples: - -* An application which reads an object structure from a file and wishes to - save it to the database might parse the file, build up the - structure, and then use - :meth:`~.Session.merge` to save it - to the database, ensuring that the data within the file is - used to formulate the primary key of each element of the - structure. Later, when the file has changed, the same - process can be re-run, producing a slightly different - object structure, which can then be ``merged`` in again, - and the :class:`~sqlalchemy.orm.session.Session` will - automatically update the database to reflect those - changes, loading each object from the database by primary key and - then updating its state with the new state given. - -* An application is storing objects in an in-memory cache, shared by - many :class:`.Session` objects simultaneously. :meth:`~.Session.merge` - is used each time an object is retrieved from the cache to create - a local copy of it in each :class:`.Session` which requests it. - The cached object remains detached; only its state is moved into - copies of itself that are local to individual :class:`~.Session` - objects. - - In the caching use case, it's common to use the ``load=False`` - flag to remove the overhead of reconciling the object's state - with the database. There's also a "bulk" version of - :meth:`~.Session.merge` called :meth:`~.Query.merge_result` - that was designed to work with cache-extended :class:`.Query` - objects - see the section :ref:`examples_caching`. - -* An application wants to transfer the state of a series of objects - into a :class:`.Session` maintained by a worker thread or other - concurrent system. :meth:`~.Session.merge` makes a copy of each object - to be placed into this new :class:`.Session`. At the end of the operation, - the parent thread/process maintains the objects it started with, - and the thread/worker can proceed with local copies of those objects. - - In the "transfer between threads/processes" use case, the application - may want to use the ``load=False`` flag as well to avoid overhead and - redundant SQL queries as the data is transferred. - -Merge Tips -~~~~~~~~~~ - -:meth:`~.Session.merge` is an extremely useful method for many purposes. However, -it deals with the intricate border between objects that are transient/detached and -those that are persistent, as well as the automated transference of state. -The wide variety of scenarios that can present themselves here often require a -more careful approach to the state of objects. Common problems with merge usually involve -some unexpected state regarding the object being passed to :meth:`~.Session.merge`. - -Lets use the canonical example of the User and Address objects:: - - class User(Base): - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - name = Column(String(50), nullable=False) - addresses = relationship("Address", backref="user") - - class Address(Base): - __tablename__ = 'address' - - id = Column(Integer, primary_key=True) - email_address = Column(String(50), nullable=False) - user_id = Column(Integer, ForeignKey('user.id'), nullable=False) - -Assume a ``User`` object with one ``Address``, already persistent:: - - >>> u1 = User(name='ed', addresses=[Address(email_address='ed@ed.com')]) - >>> session.add(u1) - >>> session.commit() - -We now create ``a1``, an object outside the session, which we'd like -to merge on top of the existing ``Address``:: - - >>> existing_a1 = u1.addresses[0] - >>> a1 = Address(id=existing_a1.id) - -A surprise would occur if we said this:: - - >>> a1.user = u1 - >>> a1 = session.merge(a1) - >>> session.commit() - sqlalchemy.orm.exc.FlushError: New instance
- with identity key (, (1,)) conflicts with - persistent instance
- -Why is that ? We weren't careful with our cascades. The assignment -of ``a1.user`` to a persistent object cascaded to the backref of ``User.addresses`` -and made our ``a1`` object pending, as though we had added it. Now we have -*two* ``Address`` objects in the session:: - - >>> a1 = Address() - >>> a1.user = u1 - >>> a1 in session - True - >>> existing_a1 in session - True - >>> a1 is existing_a1 - False - -Above, our ``a1`` is already pending in the session. The -subsequent :meth:`~.Session.merge` operation essentially -does nothing. Cascade can be configured via the :paramref:`~.relationship.cascade` -option on :func:`.relationship`, although in this case it -would mean removing the ``save-update`` cascade from the -``User.addresses`` relationship - and usually, that behavior -is extremely convenient. The solution here would usually be to not assign -``a1.user`` to an object already persistent in the target -session. - -The ``cascade_backrefs=False`` option of :func:`.relationship` -will also prevent the ``Address`` from -being added to the session via the ``a1.user = u1`` assignment. - -Further detail on cascade operation is at :ref:`unitofwork_cascades`. - -Another example of unexpected state:: - - >>> a1 = Address(id=existing_a1.id, user_id=u1.id) - >>> assert a1.user is None - >>> True - >>> a1 = session.merge(a1) - >>> session.commit() - sqlalchemy.exc.IntegrityError: (IntegrityError) address.user_id - may not be NULL - -Here, we accessed a1.user, which returned its default value -of ``None``, which as a result of this access, has been placed in the ``__dict__`` of -our object ``a1``. Normally, this operation creates no change event, -so the ``user_id`` attribute takes precedence during a -flush. But when we merge the ``Address`` object into the session, the operation -is equivalent to:: - - >>> existing_a1.id = existing_a1.id - >>> existing_a1.user_id = u1.id - >>> existing_a1.user = None - -Where above, both ``user_id`` and ``user`` are assigned to, and change events -are emitted for both. The ``user`` association -takes precedence, and None is applied to ``user_id``, causing a failure. - -Most :meth:`~.Session.merge` issues can be examined by first checking - -is the object prematurely in the session ? - -.. sourcecode:: python+sql - - >>> a1 = Address(id=existing_a1, user_id=user.id) - >>> assert a1 not in session - >>> a1 = session.merge(a1) - -Or is there state on the object that we don't want ? Examining ``__dict__`` -is a quick way to check:: - - >>> a1 = Address(id=existing_a1, user_id=user.id) - >>> a1.user - >>> a1.__dict__ - {'_sa_instance_state': , - 'user_id': 1, - 'id': 1, - 'user': None} - >>> # we don't want user=None merged, remove it - >>> del a1.user - >>> a1 = session.merge(a1) - >>> # success - >>> session.commit() - -Deleting --------- - -The :meth:`~.Session.delete` method places an instance -into the Session's list of objects to be marked as deleted:: - - # mark two objects to be deleted - session.delete(obj1) - session.delete(obj2) - - # commit (or flush) - session.commit() - -.. _session_deleting_from_collections: - -Deleting from Collections -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A common confusion that arises regarding :meth:`~.Session.delete` is when -objects which are members of a collection are being deleted. While the -collection member is marked for deletion from the database, this does not -impact the collection itself in memory until the collection is expired. -Below, we illustrate that even after an ``Address`` object is marked -for deletion, it's still present in the collection associated with the -parent ``User``, even after a flush:: - - >>> address = user.addresses[1] - >>> session.delete(address) - >>> session.flush() - >>> address in user.addresses - True - -When the above session is committed, all attributes are expired. The next -access of ``user.addresses`` will re-load the collection, revealing the -desired state:: - - >>> session.commit() - >>> address in user.addresses - False - -The usual practice of deleting items within collections is to forego the usage -of :meth:`~.Session.delete` directly, and instead use cascade behavior to -automatically invoke the deletion as a result of removing the object from -the parent collection. The ``delete-orphan`` cascade accomplishes this, -as illustrated in the example below:: - - mapper(User, users_table, properties={ - 'addresses':relationship(Address, cascade="all, delete, delete-orphan") - }) - del user.addresses[1] - session.flush() - -Where above, upon removing the ``Address`` object from the ``User.addresses`` -collection, the ``delete-orphan`` cascade has the effect of marking the ``Address`` -object for deletion in the same way as passing it to :meth:`~.Session.delete`. - -See also :ref:`unitofwork_cascades` for detail on cascades. - -Deleting based on Filter Criterion -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The caveat with ``Session.delete()`` is that you need to have an object handy -already in order to delete. The Query includes a -:func:`~sqlalchemy.orm.query.Query.delete` method which deletes based on -filtering criteria:: - - session.query(User).filter(User.id==7).delete() - -The ``Query.delete()`` method includes functionality to "expire" objects -already in the session which match the criteria. However it does have some -caveats, including that "delete" and "delete-orphan" cascades won't be fully -expressed for collections which are already loaded. See the API docs for -:meth:`~sqlalchemy.orm.query.Query.delete` for more details. - -.. _session_flushing: - -Flushing --------- - -When the :class:`~sqlalchemy.orm.session.Session` is used with its default -configuration, the flush step is nearly always done transparently. -Specifically, the flush occurs before any individual -:class:`~sqlalchemy.orm.query.Query` is issued, as well as within the -:meth:`~.Session.commit` call before the transaction is -committed. It also occurs before a SAVEPOINT is issued when -:meth:`~.Session.begin_nested` is used. - -Regardless of the autoflush setting, a flush can always be forced by issuing -:meth:`~.Session.flush`:: - - session.flush() - -The "flush-on-Query" aspect of the behavior can be disabled by constructing -:class:`.sessionmaker` with the flag ``autoflush=False``:: - - Session = sessionmaker(autoflush=False) - -Additionally, autoflush can be temporarily disabled by setting the -``autoflush`` flag at any time:: - - mysession = Session() - mysession.autoflush = False - -Some autoflush-disable recipes are available at `DisableAutoFlush -`_. - -The flush process *always* occurs within a transaction, even if the -:class:`~sqlalchemy.orm.session.Session` has been configured with -``autocommit=True``, a setting that disables the session's persistent -transactional state. If no transaction is present, -:meth:`~.Session.flush` creates its own transaction and -commits it. Any failures during flush will always result in a rollback of -whatever transaction is present. If the Session is not in ``autocommit=True`` -mode, an explicit call to :meth:`~.Session.rollback` is -required after a flush fails, even though the underlying transaction will have -been rolled back already - this is so that the overall nesting pattern of -so-called "subtransactions" is consistently maintained. - -.. _session_committing: - -Committing ----------- - -:meth:`~.Session.commit` is used to commit the current -transaction. It always issues :meth:`~.Session.flush` -beforehand to flush any remaining state to the database; this is independent -of the "autoflush" setting. If no transaction is present, it raises an error. -Note that the default behavior of the :class:`~sqlalchemy.orm.session.Session` -is that a "transaction" is always present; this behavior can be disabled by -setting ``autocommit=True``. In autocommit mode, a transaction can be -initiated by calling the :meth:`~.Session.begin` method. - -.. note:: - - The term "transaction" here refers to a transactional - construct within the :class:`.Session` itself which may be - maintaining zero or more actual database (DBAPI) transactions. An individual - DBAPI connection begins participation in the "transaction" as it is first - used to execute a SQL statement, then remains present until the session-level - "transaction" is completed. See :ref:`unitofwork_transaction` for - further detail. - -Another behavior of :meth:`~.Session.commit` is that by -default it expires the state of all instances present after the commit is -complete. This is so that when the instances are next accessed, either through -attribute access or by them being present in a -:class:`~sqlalchemy.orm.query.Query` result set, they receive the most recent -state. To disable this behavior, configure -:class:`.sessionmaker` with ``expire_on_commit=False``. - -Normally, instances loaded into the :class:`~sqlalchemy.orm.session.Session` -are never changed by subsequent queries; the assumption is that the current -transaction is isolated so the state most recently loaded is correct as long -as the transaction continues. Setting ``autocommit=True`` works against this -model to some degree since the :class:`~sqlalchemy.orm.session.Session` -behaves in exactly the same way with regard to attribute state, except no -transaction is present. - -.. _session_rollback: - -Rolling Back ------------- - -:meth:`~.Session.rollback` rolls back the current -transaction. With a default configured session, the post-rollback state of the -session is as follows: - - * All transactions are rolled back and all connections returned to the - connection pool, unless the Session was bound directly to a Connection, in - which case the connection is still maintained (but still rolled back). - * Objects which were initially in the *pending* state when they were added - to the :class:`~sqlalchemy.orm.session.Session` within the lifespan of the - transaction are expunged, corresponding to their INSERT statement being - rolled back. The state of their attributes remains unchanged. - * Objects which were marked as *deleted* within the lifespan of the - transaction are promoted back to the *persistent* state, corresponding to - their DELETE statement being rolled back. Note that if those objects were - first *pending* within the transaction, that operation takes precedence - instead. - * All objects not expunged are fully expired. - -With that state understood, the :class:`~sqlalchemy.orm.session.Session` may -safely continue usage after a rollback occurs. - -When a :meth:`~.Session.flush` fails, typically for -reasons like primary key, foreign key, or "not nullable" constraint -violations, a :meth:`~.Session.rollback` is issued -automatically (it's currently not possible for a flush to continue after a -partial failure). However, the flush process always uses its own transactional -demarcator called a *subtransaction*, which is described more fully in the -docstrings for :class:`~sqlalchemy.orm.session.Session`. What it means here is -that even though the database transaction has been rolled back, the end user -must still issue :meth:`~.Session.rollback` to fully -reset the state of the :class:`~sqlalchemy.orm.session.Session`. - -Expunging ---------- - -Expunge removes an object from the Session, sending persistent instances to -the detached state, and pending instances to the transient state: - -.. sourcecode:: python+sql - - session.expunge(obj1) - -To remove all items, call :meth:`~.Session.expunge_all` -(this method was formerly known as ``clear()``). - -Closing -------- - -The :meth:`~.Session.close` method issues a -:meth:`~.Session.expunge_all`, and :term:`releases` any -transactional/connection resources. When connections are returned to the -connection pool, transactional state is rolled back as well. - -.. _session_expire: - -Refreshing / Expiring ---------------------- - -:term:`Expiring` means that the database-persisted data held inside a series -of object attributes is erased, in such a way that when those attributes -are next accessed, a SQL query is emitted which will refresh that data from -the database. - -When we talk about expiration of data we are usually talking about an object -that is in the :term:`persistent` state. For example, if we load an object -as follows:: - - user = session.query(User).filter_by(name='user1').first() - -The above ``User`` object is persistent, and has a series of attributes -present; if we were to look inside its ``__dict__``, we'd see that state -loaded:: - - >>> user.__dict__ - { - 'id': 1, 'name': u'user1', - '_sa_instance_state': <...>, - } - -where ``id`` and ``name`` refer to those columns in the database. -``_sa_instance_state`` is a non-database-persisted value used by SQLAlchemy -internally (it refers to the :class:`.InstanceState` for the instance. -While not directly relevant to this section, if we want to get at it, -we should use the :func:`.inspect` function to access it). - -At this point, the state in our ``User`` object matches that of the loaded -database row. But upon expiring the object using a method such as -:meth:`.Session.expire`, we see that the state is removed:: - - >>> session.expire(user) - >>> user.__dict__ - {'_sa_instance_state': <...>} - -We see that while the internal "state" still hangs around, the values which -correspond to the ``id`` and ``name`` columns are gone. If we were to access -one of these columns and are watching SQL, we'd see this: - -.. sourcecode:: python+sql - - >>> print(user.name) - {opensql}SELECT user.id AS user_id, user.name AS user_name - FROM user - WHERE user.id = ? - (1,) - {stop}user1 - -Above, upon accessing the expired attribute ``user.name``, the ORM initiated -a :term:`lazy load` to retrieve the most recent state from the database, -by emitting a SELECT for the user row to which this user refers. Afterwards, -the ``__dict__`` is again populated:: - - >>> user.__dict__ - { - 'id': 1, 'name': u'user1', - '_sa_instance_state': <...>, - } - -.. note:: While we are peeking inside of ``__dict__`` in order to see a bit - of what SQLAlchemy does with object attributes, we **should not modify** - the contents of ``__dict__`` directly, at least as far as those attributes - which the SQLAlchemy ORM is maintaining (other attributes outside of SQLA's - realm are fine). This is because SQLAlchemy uses :term:`descriptors` in - order to track the changes we make to an object, and when we modify ``__dict__`` - directly, the ORM won't be able to track that we changed something. - -Another key behavior of both :meth:`~.Session.expire` and :meth:`~.Session.refresh` -is that all un-flushed changes on an object are discarded. That is, -if we were to modify an attribute on our ``User``:: - - >>> user.name = 'user2' - -but then we call :meth:`~.Session.expire` without first calling :meth:`~.Session.flush`, -our pending value of ``'user2'`` is discarded:: - - >>> session.expire(user) - >>> user.name - 'user1' - -The :meth:`~.Session.expire` method can be used to mark as "expired" all ORM-mapped -attributes for an instance:: - - # expire all ORM-mapped attributes on obj1 - session.expire(obj1) - -it can also be passed a list of string attribute names, referring to specific -attributes to be marked as expired:: - - # expire only attributes obj1.attr1, obj1.attr2 - session.expire(obj1, ['attr1', 'attr2']) - -The :meth:`~.Session.refresh` method has a similar interface, but instead -of expiring, it emits an immediate SELECT for the object's row immediately:: - - # reload all attributes on obj1 - session.refresh(obj1) - -:meth:`~.Session.refresh` also accepts a list of string attribute names, -but unlike :meth:`~.Session.expire`, expects at least one name to -be that of a column-mapped attribute:: - - # reload obj1.attr1, obj1.attr2 - session.refresh(obj1, ['attr1', 'attr2']) - -The :meth:`.Session.expire_all` method allows us to essentially call -:meth:`.Session.expire` on all objects contained within the :class:`.Session` -at once:: - - session.expire_all() - -What Actually Loads -~~~~~~~~~~~~~~~~~~~ - -The SELECT statement that's emitted when an object marked with :meth:`~.Session.expire` -or loaded with :meth:`~.Session.refresh` varies based on several factors, including: - -* The load of expired attributes is triggered from **column-mapped attributes only**. - While any kind of attribute can be marked as expired, including a - :func:`.relationship` - mapped attribute, accessing an expired :func:`.relationship` - attribute will emit a load only for that attribute, using standard - relationship-oriented lazy loading. Column-oriented attributes, even if - expired, will not load as part of this operation, and instead will load when - any column-oriented attribute is accessed. - -* :func:`.relationship`- mapped attributes will not load in response to - expired column-based attributes being accessed. - -* Regarding relationships, :meth:`~.Session.refresh` is more restrictive than - :meth:`~.Session.expire` with regards to attributes that aren't column-mapped. - Calling :meth:`.refresh` and passing a list of names that only includes - relationship-mapped attributes will actually raise an error. - In any case, non-eager-loading :func:`.relationship` attributes will not be - included in any refresh operation. - -* :func:`.relationship` attributes configured as "eager loading" via the - :paramref:`~.relationship.lazy` parameter will load in the case of - :meth:`~.Session.refresh`, if either no attribute names are specified, or - if their names are inclued in the list of attributes to be - refreshed. - -* Attributes that are configured as :func:`.deferred` will not normally load, - during either the expired-attribute load or during a refresh. - An unloaded attribute that's :func:`.deferred` instead loads on its own when directly - accessed, or if part of a "group" of deferred attributes where an unloaded - attribute in that group is accessed. - -* For expired attributes that are loaded on access, a joined-inheritance table - mapping will emit a SELECT that typically only includes those tables for which - unloaded attributes are present. The action here is sophisticated enough - to load only the parent or child table, for example, if the subset of columns - that were originally expired encompass only one or the other of those tables. - -* When :meth:`~.Session.refresh` is used on a joined-inheritance table mapping, - the SELECT emitted will resemble that of when :meth:`.Session.query` is - used on the target object's class. This is typically all those tables that - are set up as part of the mapping. - - -When to Expire or Refresh -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`.Session` uses the expiration feature automatically whenever -the transaction referred to by the session ends. Meaning, whenever :meth:`.Session.commit` -or :meth:`.Session.rollback` is called, all objects within the :class:`.Session` -are expired, using a feature equivalent to that of the :meth:`.Session.expire_all` -method. The rationale is that the end of a transaction is a -demarcating point at which there is no more context available in order to know -what the current state of the database is, as any number of other transactions -may be affecting it. Only when a new transaction starts can we again have access -to the current state of the database, at which point any number of changes -may have occurred. - -.. sidebar:: Transaction Isolation - - Of course, most databases are capable of handling - multiple transactions at once, even involving the same rows of data. When - a relational database handles multiple transactions involving the same - tables or rows, this is when the :term:`isolation` aspect of the database comes - into play. The isolation behavior of different databases varies considerably - and even on a single database can be configured to behave in different ways - (via the so-called :term:`isolation level` setting). In that sense, the :class:`.Session` - can't fully predict when the same SELECT statement, emitted a second time, - will definitely return the data we already have, or will return new data. - So as a best guess, it assumes that within the scope of a transaction, unless - it is known that a SQL expression has been emitted to modify a particular row, - there's no need to refresh a row unless explicitly told to do so. - -The :meth:`.Session.expire` and :meth:`.Session.refresh` methods are used in -those cases when one wants to force an object to re-load its data from the -database, in those cases when it is known that the current state of data -is possibly stale. Reasons for this might include: - -* some SQL has been emitted within the transaction outside of the - scope of the ORM's object handling, such as if a :meth:`.Table.update` construct - were emitted using the :meth:`.Session.execute` method; - -* if the application - is attempting to acquire data that is known to have been modified in a - concurrent transaction, and it is also known that the isolation rules in effect - allow this data to be visible. - -The second bullet has the important caveat that "it is also known that the isolation rules in effect -allow this data to be visible." This means that it cannot be assumed that an -UPDATE that happened on another database connection will yet be visible here -locally; in many cases, it will not. This is why if one wishes to use -:meth:`.expire` or :meth:`.refresh` in order to view data between ongoing -transactions, an understanding of the isolation behavior in effect is essential. - -.. seealso:: - - :meth:`.Session.expire` - - :meth:`.Session.expire_all` - - :meth:`.Session.refresh` - - :term:`isolation` - glossary explanation of isolation which includes links - to Wikipedia. - - `The SQLAlchemy Session In-Depth `_ - a video + slides with an in-depth discussion of the object - lifecycle including the role of data expiration. - - -Session Attributes ------------------- - -The :class:`~sqlalchemy.orm.session.Session` itself acts somewhat like a -set-like collection. All items present may be accessed using the iterator -interface:: - - for obj in session: - print obj - -And presence may be tested for using regular "contains" semantics:: - - if obj in session: - print "Object is present" - -The session is also keeping track of all newly created (i.e. pending) objects, -all objects which have had changes since they were last loaded or saved (i.e. -"dirty"), and everything that's been marked as deleted:: - - # pending objects recently added to the Session - session.new - - # persistent objects which currently have changes detected - # (this collection is now created on the fly each time the property is called) - session.dirty - - # persistent objects that have been marked as deleted via session.delete(obj) - session.deleted - - # dictionary of all persistent objects, keyed on their - # identity key - session.identity_map - -(Documentation: :attr:`.Session.new`, :attr:`.Session.dirty`, -:attr:`.Session.deleted`, :attr:`.Session.identity_map`). - -Note that objects within the session are by default *weakly referenced*. This -means that when they are dereferenced in the outside application, they fall -out of scope from within the :class:`~sqlalchemy.orm.session.Session` as well -and are subject to garbage collection by the Python interpreter. The -exceptions to this include objects which are pending, objects which are marked -as deleted, or persistent objects which have pending changes on them. After a -full flush, these collections are all empty, and all objects are again weakly -referenced. To disable the weak referencing behavior and force all objects -within the session to remain until explicitly expunged, configure -:class:`.sessionmaker` with the ``weak_identity_map=False`` -setting. - -.. _unitofwork_cascades: - -Cascades -======== - -Mappers support the concept of configurable :term:`cascade` behavior on -:func:`~sqlalchemy.orm.relationship` constructs. This refers -to how operations performed on a "parent" object relative to a -particular :class:`.Session` should be propagated to items -referred to by that relationship (e.g. "child" objects), and is -affected by the :paramref:`.relationship.cascade` option. - -The default behavior of cascade is limited to cascades of the -so-called :ref:`cascade_save_update` and :ref:`cascade_merge` settings. -The typical "alternative" setting for cascade is to add -the :ref:`cascade_delete` and :ref:`cascade_delete_orphan` options; -these settings are appropriate for related objects which only exist as -long as they are attached to their parent, and are otherwise deleted. - -Cascade behavior is configured using the by changing the -:paramref:`~.relationship.cascade` option on -:func:`~sqlalchemy.orm.relationship`:: - - class Order(Base): - __tablename__ = 'order' - - items = relationship("Item", cascade="all, delete-orphan") - customer = relationship("User", cascade="save-update") - -To set cascades on a backref, the same flag can be used with the -:func:`~.sqlalchemy.orm.backref` function, which ultimately feeds -its arguments back into :func:`~sqlalchemy.orm.relationship`:: - - class Item(Base): - __tablename__ = 'item' - - order = relationship("Order", - backref=backref("items", cascade="all, delete-orphan") - ) - -.. sidebar:: The Origins of Cascade - - SQLAlchemy's notion of cascading behavior on relationships, - as well as the options to configure them, are primarily derived - from the similar feature in the Hibernate ORM; Hibernate refers - to "cascade" in a few places such as in - `Example: Parent/Child `_. - If cascades are confusing, we'll refer to their conclusion, - stating "The sections we have just covered can be a bit confusing. - However, in practice, it all works out nicely." - -The default value of :paramref:`~.relationship.cascade` is ``save-update, merge``. -The typical alternative setting for this parameter is either -``all`` or more commonly ``all, delete-orphan``. The ``all`` symbol -is a synonym for ``save-update, merge, refresh-expire, expunge, delete``, -and using it in conjunction with ``delete-orphan`` indicates that the child -object should follow along with its parent in all cases, and be deleted once -it is no longer associated with that parent. - -The list of available values which can be specified for -the :paramref:`~.relationship.cascade` parameter are described in the following subsections. - -.. _cascade_save_update: - -save-update ------------ - -``save-update`` cascade indicates that when an object is placed into a -:class:`.Session` via :meth:`.Session.add`, all the objects associated -with it via this :func:`.relationship` should also be added to that -same :class:`.Session`. Suppose we have an object ``user1`` with two -related objects ``address1``, ``address2``:: - - >>> user1 = User() - >>> address1, address2 = Address(), Address() - >>> user1.addresses = [address1, address2] - -If we add ``user1`` to a :class:`.Session`, it will also add -``address1``, ``address2`` implicitly:: - - >>> sess = Session() - >>> sess.add(user1) - >>> address1 in sess - True - -``save-update`` cascade also affects attribute operations for objects -that are already present in a :class:`.Session`. If we add a third -object, ``address3`` to the ``user1.addresses`` collection, it -becomes part of the state of that :class:`.Session`:: - - >>> address3 = Address() - >>> user1.append(address3) - >>> address3 in sess - >>> True - -``save-update`` has the possibly surprising behavior which is that -persistent objects which were *removed* from a collection -or in some cases a scalar attribute -may also be pulled into the :class:`.Session` of a parent object; this is -so that the flush process may handle that related object appropriately. -This case can usually only arise if an object is removed from one :class:`.Session` -and added to another:: - - >>> user1 = sess1.query(User).filter_by(id=1).first() - >>> address1 = user1.addresses[0] - >>> sess1.close() # user1, address1 no longer associated with sess1 - >>> user1.addresses.remove(address1) # address1 no longer associated with user1 - >>> sess2 = Session() - >>> sess2.add(user1) # ... but it still gets added to the new session, - >>> address1 in sess2 # because it's still "pending" for flush - True - -The ``save-update`` cascade is on by default, and is typically taken -for granted; it simplifies code by allowing a single call to -:meth:`.Session.add` to register an entire structure of objects within -that :class:`.Session` at once. While it can be disabled, there -is usually not a need to do so. - -One case where ``save-update`` cascade does sometimes get in the way is in that -it takes place in both directions for bi-directional relationships, e.g. -backrefs, meaning that the association of a child object with a particular parent -can have the effect of the parent object being implicitly associated with that -child object's :class:`.Session`; this pattern, as well as how to modify its -behavior using the :paramref:`~.relationship.cascade_backrefs` flag, -is discussed in the section :ref:`backref_cascade`. - -.. _cascade_delete: - -delete ------- - -The ``delete`` cascade indicates that when a "parent" object -is marked for deletion, its related "child" objects should also be marked -for deletion. If for example we we have a relationship ``User.addresses`` -with ``delete`` cascade configured:: - - class User(Base): - # ... - - addresses = relationship("Address", cascade="save-update, merge, delete") - -If using the above mapping, we have a ``User`` object and two -related ``Address`` objects:: - - >>> user1 = sess.query(User).filter_by(id=1).first() - >>> address1, address2 = user1.addresses - -If we mark ``user1`` for deletion, after the flush operation proceeds, -``address1`` and ``address2`` will also be deleted: - -.. sourcecode:: python+sql - - >>> sess.delete(user1) - >>> sess.commit() - {opensql}DELETE FROM address WHERE address.id = ? - ((1,), (2,)) - DELETE FROM user WHERE user.id = ? - (1,) - COMMIT - -Alternatively, if our ``User.addresses`` relationship does *not* have -``delete`` cascade, SQLAlchemy's default behavior is to instead de-associate -``address1`` and ``address2`` from ``user1`` by setting their foreign key -reference to ``NULL``. Using a mapping as follows:: - - class User(Base): - # ... - - addresses = relationship("Address") - -Upon deletion of a parent ``User`` object, the rows in ``address`` are not -deleted, but are instead de-associated: - -.. sourcecode:: python+sql - - >>> sess.delete(user1) - >>> sess.commit() - {opensql}UPDATE address SET user_id=? WHERE address.id = ? - (None, 1) - UPDATE address SET user_id=? WHERE address.id = ? - (None, 2) - DELETE FROM user WHERE user.id = ? - (1,) - COMMIT - -``delete`` cascade is more often than not used in conjunction with -:ref:`cascade_delete_orphan` cascade, which will emit a DELETE for the related -row if the "child" object is deassociated from the parent. The combination -of ``delete`` and ``delete-orphan`` cascade covers both situations where -SQLAlchemy has to decide between setting a foreign key column to NULL versus -deleting the row entirely. - -.. topic:: ORM-level "delete" cascade vs. FOREIGN KEY level "ON DELETE" cascade - - The behavior of SQLAlchemy's "delete" cascade has a lot of overlap with the - ``ON DELETE CASCADE`` feature of a database foreign key, as well - as with that of the ``ON DELETE SET NULL`` foreign key setting when "delete" - cascade is not specified. Database level "ON DELETE" cascades are specific to the - "FOREIGN KEY" construct of the relational database; SQLAlchemy allows - configuration of these schema-level constructs at the :term:`DDL` level - using options on :class:`.ForeignKeyConstraint` which are described - at :ref:`on_update_on_delete`. - - It is important to note the differences between the ORM and the relational - database's notion of "cascade" as well as how they integrate: - - * A database level ``ON DELETE`` cascade is configured effectively - on the **many-to-one** side of the relationship; that is, we configure - it relative to the ``FOREIGN KEY`` constraint that is the "many" side - of a relationship. At the ORM level, **this direction is reversed**. - SQLAlchemy handles the deletion of "child" objects relative to a - "parent" from the "parent" side, which means that ``delete`` and - ``delete-orphan`` cascade are configured on the **one-to-many** - side. - - * Database level foreign keys with no ``ON DELETE`` setting - are often used to **prevent** a parent - row from being removed, as it would necessarily leave an unhandled - related row present. If this behavior is desired in a one-to-many - relationship, SQLAlchemy's default behavior of setting a foreign key - to ``NULL`` can be caught in one of two ways: - - * The easiest and most common is just to set the - foreign-key-holding column to ``NOT NULL`` at the database schema - level. An attempt by SQLAlchemy to set the column to NULL will - fail with a simple NOT NULL constraint exception. - - * The other, more special case way is to set the :paramref:`~.relationship.passive_deletes` - flag to the string ``"all"``. This has the effect of entirely - disabling SQLAlchemy's behavior of setting the foreign key column - to NULL, and a DELETE will be emitted for the parent row without - any affect on the child row, even if the child row is present - in memory. This may be desirable in the case when - database-level foreign key triggers, either special ``ON DELETE`` settings - or otherwise, need to be activated in all cases when a parent row is deleted. - - * Database level ``ON DELETE`` cascade is **vastly more efficient** - than that of SQLAlchemy. The database can chain a series of cascade - operations across many relationships at once; e.g. if row A is deleted, - all the related rows in table B can be deleted, and all the C rows related - to each of those B rows, and on and on, all within the scope of a single - DELETE statement. SQLAlchemy on the other hand, in order to support - the cascading delete operation fully, has to individually load each - related collection in order to target all rows that then may have further - related collections. That is, SQLAlchemy isn't sophisticated enough - to emit a DELETE for all those related rows at once within this context. - - * SQLAlchemy doesn't **need** to be this sophisticated, as we instead provide - smooth integration with the database's own ``ON DELETE`` functionality, - by using the :paramref:`~.relationship.passive_deletes` option in conjunction - with properly configured foreign key constraints. Under this behavior, - SQLAlchemy only emits DELETE for those rows that are already locally - present in the :class:`.Session`; for any collections that are unloaded, - it leaves them to the database to handle, rather than emitting a SELECT - for them. The section :ref:`passive_deletes` provides an example of this use. - - * While database-level ``ON DELETE`` functionality works only on the "many" - side of a relationship, SQLAlchemy's "delete" cascade - has **limited** ability to operate in the *reverse* direction as well, - meaning it can be configured on the "many" side to delete an object - on the "one" side when the reference on the "many" side is deleted. However - this can easily result in constraint violations if there are other objects - referring to this "one" side from the "many", so it typically is only - useful when a relationship is in fact a "one to one". The - :paramref:`~.relationship.single_parent` flag should be used to establish - an in-Python assertion for this case. - - -When using a :func:`.relationship` that also includes a many-to-many -table using the :paramref:`~.relationship.secondary` option, SQLAlchemy's -delete cascade handles the rows in this many-to-many table automatically. -Just like, as described in :ref:`relationships_many_to_many_deletion`, -the addition or removal of an object from a many-to-many collection -results in the INSERT or DELETE of a row in the many-to-many table, -the ``delete`` cascade, when activated as the result of a parent object -delete operation, will DELETE not just the row in the "child" table but also -in the many-to-many table. - -.. _cascade_delete_orphan: - -delete-orphan -------------- - -``delete-orphan`` cascade adds behavior to the ``delete`` cascade, -such that a child object will be marked for deletion when it is -de-associated from the parent, not just when the parent is marked -for deletion. This is a common feature when dealing with a related -object that is "owned" by its parent, with a NOT NULL foreign key, -so that removal of the item from the parent collection results -in its deletion. - -``delete-orphan`` cascade implies that each child object can only -have one parent at a time, so is configured in the vast majority of cases -on a one-to-many relationship. Setting it on a many-to-one or -many-to-many relationship is more awkward; for this use case, -SQLAlchemy requires that the :func:`~sqlalchemy.orm.relationship` -be configured with the :paramref:`~.relationship.single_parent` argument, -establishes Python-side validation that ensures the object -is associated with only one parent at a time. - -.. _cascade_merge: - -merge ------ - -``merge`` cascade indicates that the :meth:`.Session.merge` -operation should be propagated from a parent that's the subject -of the :meth:`.Session.merge` call down to referred objects. -This cascade is also on by default. - -.. _cascade_refresh_expire: - -refresh-expire --------------- - -``refresh-expire`` is an uncommon option, indicating that the -:meth:`.Session.expire` operation should be propagated from a parent -down to referred objects. When using :meth:`.Session.refresh`, -the referred objects are expired only, but not actually refreshed. - -.. _cascade_expunge: - -expunge -------- - -``expunge`` cascade indicates that when the parent object is removed -from the :class:`.Session` using :meth:`.Session.expunge`, the -operation should be propagated down to referred objects. - -.. _backref_cascade: - -Controlling Cascade on Backrefs -------------------------------- - -The :ref:`cascade_save_update` cascade by default takes place on attribute change events -emitted from backrefs. This is probably a confusing statement more -easily described through demonstration; it means that, given a mapping such as this:: - - mapper(Order, order_table, properties={ - 'items' : relationship(Item, backref='order') - }) - -If an ``Order`` is already in the session, and is assigned to the ``order`` -attribute of an ``Item``, the backref appends the ``Order`` to the ``items`` -collection of that ``Order``, resulting in the ``save-update`` cascade taking -place:: - - >>> o1 = Order() - >>> session.add(o1) - >>> o1 in session - True - - >>> i1 = Item() - >>> i1.order = o1 - >>> i1 in o1.items - True - >>> i1 in session - True - -This behavior can be disabled using the :paramref:`~.relationship.cascade_backrefs` flag:: - - mapper(Order, order_table, properties={ - 'items' : relationship(Item, backref='order', - cascade_backrefs=False) - }) - -So above, the assignment of ``i1.order = o1`` will append ``i1`` to the ``items`` -collection of ``o1``, but will not add ``i1`` to the session. You can, of -course, :meth:`~.Session.add` ``i1`` to the session at a later point. This -option may be helpful for situations where an object needs to be kept out of a -session until it's construction is completed, but still needs to be given -associations to objects which are already persistent in the target session. - - -.. _unitofwork_transaction: - -Managing Transactions -===================== - -A newly constructed :class:`.Session` may be said to be in the "begin" state. -In this state, the :class:`.Session` has not established any connection or -transactional state with any of the :class:`.Engine` objects that may be associated -with it. - -The :class:`.Session` then receives requests to operate upon a database connection. -Typically, this means it is called upon to execute SQL statements using a particular -:class:`.Engine`, which may be via :meth:`.Session.query`, :meth:`.Session.execute`, -or within a flush operation of pending data, which occurs when such state exists -and :meth:`.Session.commit` or :meth:`.Session.flush` is called. - -As these requests are received, each new :class:`.Engine` encountered is associated -with an ongoing transactional state maintained by the :class:`.Session`. -When the first :class:`.Engine` is operated upon, the :class:`.Session` can be said -to have left the "begin" state and entered "transactional" state. For each -:class:`.Engine` encountered, a :class:`.Connection` is associated with it, -which is acquired via the :meth:`.Engine.contextual_connect` method. If a -:class:`.Connection` was directly associated with the :class:`.Session` (see :ref:`session_external_transaction` -for an example of this), it is -added to the transactional state directly. - -For each :class:`.Connection`, the :class:`.Session` also maintains a :class:`.Transaction` object, -which is acquired by calling :meth:`.Connection.begin` on each :class:`.Connection`, -or if the :class:`.Session` -object has been established using the flag ``twophase=True``, a :class:`.TwoPhaseTransaction` -object acquired via :meth:`.Connection.begin_twophase`. These transactions are all committed or -rolled back corresponding to the invocation of the -:meth:`.Session.commit` and :meth:`.Session.rollback` methods. A commit operation will -also call the :meth:`.TwoPhaseTransaction.prepare` method on all transactions if applicable. - -When the transactional state is completed after a rollback or commit, the :class:`.Session` -:term:`releases` all :class:`.Transaction` and :class:`.Connection` resources, -and goes back to the "begin" state, which -will again invoke new :class:`.Connection` and :class:`.Transaction` objects as new -requests to emit SQL statements are received. - -The example below illustrates this lifecycle:: - - engine = create_engine("...") - Session = sessionmaker(bind=engine) - - # new session. no connections are in use. - session = Session() - try: - # first query. a Connection is acquired - # from the Engine, and a Transaction - # started. - item1 = session.query(Item).get(1) - - # second query. the same Connection/Transaction - # are used. - item2 = session.query(Item).get(2) - - # pending changes are created. - item1.foo = 'bar' - item2.bar = 'foo' - - # commit. The pending changes above - # are flushed via flush(), the Transaction - # is committed, the Connection object closed - # and discarded, the underlying DBAPI connection - # returned to the connection pool. - session.commit() - except: - # on rollback, the same closure of state - # as that of commit proceeds. - session.rollback() - raise - -.. _session_begin_nested: - -Using SAVEPOINT ---------------- - -SAVEPOINT transactions, if supported by the underlying engine, may be -delineated using the :meth:`~.Session.begin_nested` -method:: - - Session = sessionmaker() - session = Session() - session.add(u1) - session.add(u2) - - session.begin_nested() # establish a savepoint - session.add(u3) - session.rollback() # rolls back u3, keeps u1 and u2 - - session.commit() # commits u1 and u2 - -:meth:`~.Session.begin_nested` may be called any number -of times, which will issue a new SAVEPOINT with a unique identifier for each -call. For each :meth:`~.Session.begin_nested` call, a -corresponding :meth:`~.Session.rollback` or -:meth:`~.Session.commit` must be issued. (But note that if the return value is -used as a context manager, i.e. in a with-statement, then this rollback/commit -is issued by the context manager upon exiting the context, and so should not be -added explicitly.) - -When :meth:`~.Session.begin_nested` is called, a -:meth:`~.Session.flush` is unconditionally issued -(regardless of the ``autoflush`` setting). This is so that when a -:meth:`~.Session.rollback` occurs, the full state of the -session is expired, thus causing all subsequent attribute/instance access to -reference the full state of the :class:`~sqlalchemy.orm.session.Session` right -before :meth:`~.Session.begin_nested` was called. - -:meth:`~.Session.begin_nested`, in the same manner as the less often -used :meth:`~.Session.begin` method, returns a transactional object -which also works as a context manager. -It can be succinctly used around individual record inserts in order to catch -things like unique constraint exceptions:: - - for record in records: - try: - with session.begin_nested(): - session.merge(record) - except: - print "Skipped record %s" % record - session.commit() - -.. _session_autocommit: - -Autocommit Mode ---------------- - -The example of :class:`.Session` transaction lifecycle illustrated at -the start of :ref:`unitofwork_transaction` applies to a :class:`.Session` configured in the -default mode of ``autocommit=False``. Constructing a :class:`.Session` -with ``autocommit=True`` produces a :class:`.Session` placed into "autocommit" mode, where each SQL statement -invoked by a :meth:`.Session.query` or :meth:`.Session.execute` occurs -using a new connection from the connection pool, discarding it after -results have been iterated. The :meth:`.Session.flush` operation -still occurs within the scope of a single transaction, though this transaction -is closed out after the :meth:`.Session.flush` operation completes. - -.. warning:: - - "autocommit" mode should **not be considered for general use**. - If used, it should always be combined with the usage of - :meth:`.Session.begin` and :meth:`.Session.commit`, to ensure - a transaction demarcation. - - Executing queries outside of a demarcated transaction is a legacy mode - of usage, and can in some cases lead to concurrent connection - checkouts. - - In the absence of a demarcated transaction, the :class:`.Session` - cannot make appropriate decisions as to when autoflush should - occur nor when auto-expiration should occur, so these features - should be disabled with ``autoflush=False, expire_on_commit=False``. - -Modern usage of "autocommit" is for framework integrations that need to control -specifically when the "begin" state occurs. A session which is configured with -``autocommit=True`` may be placed into the "begin" state using the -:meth:`.Session.begin` method. -After the cycle completes upon :meth:`.Session.commit` or :meth:`.Session.rollback`, -connection and transaction resources are :term:`released` and the :class:`.Session` -goes back into "autocommit" mode, until :meth:`.Session.begin` is called again:: - - Session = sessionmaker(bind=engine, autocommit=True) - session = Session() - session.begin() - try: - item1 = session.query(Item).get(1) - item2 = session.query(Item).get(2) - item1.foo = 'bar' - item2.bar = 'foo' - session.commit() - except: - session.rollback() - raise - -The :meth:`.Session.begin` method also returns a transactional token which is -compatible with the Python 2.6 ``with`` statement:: - - Session = sessionmaker(bind=engine, autocommit=True) - session = Session() - with session.begin(): - item1 = session.query(Item).get(1) - item2 = session.query(Item).get(2) - item1.foo = 'bar' - item2.bar = 'foo' - -.. _session_subtransactions: - -Using Subtransactions with Autocommit -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A subtransaction indicates usage of the :meth:`.Session.begin` method in conjunction with -the ``subtransactions=True`` flag. This produces a non-transactional, delimiting construct that -allows nesting of calls to :meth:`~.Session.begin` and :meth:`~.Session.commit`. -Its purpose is to allow the construction of code that can function within a transaction -both independently of any external code that starts a transaction, -as well as within a block that has already demarcated a transaction. - -``subtransactions=True`` is generally only useful in conjunction with -autocommit, and is equivalent to the pattern described at :ref:`connections_nested_transactions`, -where any number of functions can call :meth:`.Connection.begin` and :meth:`.Transaction.commit` -as though they are the initiator of the transaction, but in fact may be participating -in an already ongoing transaction:: - - # method_a starts a transaction and calls method_b - def method_a(session): - session.begin(subtransactions=True) - try: - method_b(session) - session.commit() # transaction is committed here - except: - session.rollback() # rolls back the transaction - raise - - # method_b also starts a transaction, but when - # called from method_a participates in the ongoing - # transaction. - def method_b(session): - session.begin(subtransactions=True) - try: - session.add(SomeObject('bat', 'lala')) - session.commit() # transaction is not committed yet - except: - session.rollback() # rolls back the transaction, in this case - # the one that was initiated in method_a(). - raise - - # create a Session and call method_a - session = Session(autocommit=True) - method_a(session) - session.close() - -Subtransactions are used by the :meth:`.Session.flush` process to ensure that the -flush operation takes place within a transaction, regardless of autocommit. When -autocommit is disabled, it is still useful in that it forces the :class:`.Session` -into a "pending rollback" state, as a failed flush cannot be resumed in mid-operation, -where the end user still maintains the "scope" of the transaction overall. - -.. _session_twophase: - -Enabling Two-Phase Commit -------------------------- - -For backends which support two-phase operaration (currently MySQL and -PostgreSQL), the session can be instructed to use two-phase commit semantics. -This will coordinate the committing of transactions across databases so that -the transaction is either committed or rolled back in all databases. You can -also :meth:`~.Session.prepare` the session for -interacting with transactions not managed by SQLAlchemy. To use two phase -transactions set the flag ``twophase=True`` on the session:: - - engine1 = create_engine('postgresql://db1') - engine2 = create_engine('postgresql://db2') - - Session = sessionmaker(twophase=True) - - # bind User operations to engine 1, Account operations to engine 2 - Session.configure(binds={User:engine1, Account:engine2}) - - session = Session() - - # .... work with accounts and users - - # commit. session will issue a flush to all DBs, and a prepare step to all DBs, - # before committing both transactions - session.commit() - -.. _flush_embedded_sql_expressions: - -Embedding SQL Insert/Update Expressions into a Flush -===================================================== - -This feature allows the value of a database column to be set to a SQL -expression instead of a literal value. It's especially useful for atomic -updates, calling stored procedures, etc. All you do is assign an expression to -an attribute:: - - class SomeClass(object): - pass - mapper(SomeClass, some_table) - - someobject = session.query(SomeClass).get(5) - - # set 'value' attribute to a SQL expression adding one - someobject.value = some_table.c.value + 1 - - # issues "UPDATE some_table SET value=value+1" - session.commit() - -This technique works both for INSERT and UPDATE statements. After the -flush/commit operation, the ``value`` attribute on ``someobject`` above is -expired, so that when next accessed the newly generated value will be loaded -from the database. - -.. _session_sql_expressions: - -Using SQL Expressions with Sessions -==================================== - -SQL expressions and strings can be executed via the -:class:`~sqlalchemy.orm.session.Session` within its transactional context. -This is most easily accomplished using the -:meth:`~.Session.execute` method, which returns a -:class:`~sqlalchemy.engine.ResultProxy` in the same manner as an -:class:`~sqlalchemy.engine.Engine` or -:class:`~sqlalchemy.engine.Connection`:: - - Session = sessionmaker(bind=engine) - session = Session() - - # execute a string statement - result = session.execute("select * from table where id=:id", {'id':7}) - - # execute a SQL expression construct - result = session.execute(select([mytable]).where(mytable.c.id==7)) - -The current :class:`~sqlalchemy.engine.Connection` held by the -:class:`~sqlalchemy.orm.session.Session` is accessible using the -:meth:`~.Session.connection` method:: - - connection = session.connection() - -The examples above deal with a :class:`~sqlalchemy.orm.session.Session` that's -bound to a single :class:`~sqlalchemy.engine.Engine` or -:class:`~sqlalchemy.engine.Connection`. To execute statements using a -:class:`~sqlalchemy.orm.session.Session` which is bound either to multiple -engines, or none at all (i.e. relies upon bound metadata), both -:meth:`~.Session.execute` and -:meth:`~.Session.connection` accept a ``mapper`` keyword -argument, which is passed a mapped class or -:class:`~sqlalchemy.orm.mapper.Mapper` instance, which is used to locate the -proper context for the desired engine:: - - Session = sessionmaker() - session = Session() - - # need to specify mapper or class when executing - result = session.execute("select * from table where id=:id", {'id':7}, mapper=MyMappedClass) - - result = session.execute(select([mytable], mytable.c.id==7), mapper=MyMappedClass) - - connection = session.connection(MyMappedClass) - -.. _session_external_transaction: - -Joining a Session into an External Transaction (such as for test suites) -======================================================================== - -If a :class:`.Connection` is being used which is already in a transactional -state (i.e. has a :class:`.Transaction` established), a :class:`.Session` can -be made to participate within that transaction by just binding the -:class:`.Session` to that :class:`.Connection`. The usual rationale for this -is a test suite that allows ORM code to work freely with a :class:`.Session`, -including the ability to call :meth:`.Session.commit`, where afterwards the -entire database interaction is rolled back:: - - from sqlalchemy.orm import sessionmaker - from sqlalchemy import create_engine - from unittest import TestCase - - # global application scope. create Session class, engine - Session = sessionmaker() - - engine = create_engine('postgresql://...') - - class SomeTest(TestCase): - def setUp(self): - # connect to the database - self.connection = engine.connect() - - # begin a non-ORM transaction - self.trans = self.connection.begin() - - # bind an individual Session to the connection - self.session = Session(bind=self.connection) - - def test_something(self): - # use the session in tests. - - self.session.add(Foo()) - self.session.commit() - - def tearDown(self): - self.session.close() - - # rollback - everything that happened with the - # Session above (including calls to commit()) - # is rolled back. - self.trans.rollback() - - # return connection to the Engine - self.connection.close() - -Above, we issue :meth:`.Session.commit` as well as -:meth:`.Transaction.rollback`. This is an example of where we take advantage -of the :class:`.Connection` object's ability to maintain *subtransactions*, or -nested begin/commit-or-rollback pairs where only the outermost begin/commit -pair actually commits the transaction, or if the outermost block rolls back, -everything is rolled back. - -.. topic:: Supporting Tests with Rollbacks - - The above recipe works well for any kind of database enabled test, except - for a test that needs to actually invoke :meth:`.Session.rollback` within - the scope of the test itself. The above recipe can be expanded, such - that the :class:`.Session` always runs all operations within the scope - of a SAVEPOINT, which is established at the start of each transaction, - so that tests can also rollback the "transaction" as well while still - remaining in the scope of a larger "transaction" that's never committed, - using two extra events:: - - from sqlalchemy import event - - class SomeTest(TestCase): - def setUp(self): - # connect to the database - self.connection = engine.connect() - - # begin a non-ORM transaction - self.trans = connection.begin() - - # bind an individual Session to the connection - self.session = Session(bind=self.connection) - - # start the session in a SAVEPOINT... - self.session.begin_nested() - - # then each time that SAVEPOINT ends, reopen it - @event.listens_for(self.session, "after_transaction_end") - def restart_savepoint(session, transaction): - if transaction.nested and not transaction._parent.nested: - session.begin_nested() - - - # ... the tearDown() method stays the same - -.. _unitofwork_contextual: - -Contextual/Thread-local Sessions -================================= - -Recall from the section :ref:`session_faq_whentocreate`, the concept of -"session scopes" was introduced, with an emphasis on web applications -and the practice of linking the scope of a :class:`.Session` with that -of a web request. Most modern web frameworks include integration tools -so that the scope of the :class:`.Session` can be managed automatically, -and these tools should be used as they are available. - -SQLAlchemy includes its own helper object, which helps with the establishment -of user-defined :class:`.Session` scopes. It is also used by third-party -integration systems to help construct their integration schemes. - -The object is the :class:`.scoped_session` object, and it represents a -**registry** of :class:`.Session` objects. If you're not familiar with the -registry pattern, a good introduction can be found in `Patterns of Enterprise -Architecture `_. - -.. note:: - - The :class:`.scoped_session` object is a very popular and useful object - used by many SQLAlchemy applications. However, it is important to note - that it presents **only one approach** to the issue of :class:`.Session` - management. If you're new to SQLAlchemy, and especially if the - term "thread-local variable" seems strange to you, we recommend that - if possible you familiarize first with an off-the-shelf integration - system such as `Flask-SQLAlchemy `_ - or `zope.sqlalchemy `_. - -A :class:`.scoped_session` is constructed by calling it, passing it a -**factory** which can create new :class:`.Session` objects. A factory -is just something that produces a new object when called, and in the -case of :class:`.Session`, the most common factory is the :class:`.sessionmaker`, -introduced earlier in this section. Below we illustrate this usage:: - - >>> from sqlalchemy.orm import scoped_session - >>> from sqlalchemy.orm import sessionmaker - - >>> session_factory = sessionmaker(bind=some_engine) - >>> Session = scoped_session(session_factory) - -The :class:`.scoped_session` object we've created will now call upon the -:class:`.sessionmaker` when we "call" the registry:: - - >>> some_session = Session() - -Above, ``some_session`` is an instance of :class:`.Session`, which we -can now use to talk to the database. This same :class:`.Session` is also -present within the :class:`.scoped_session` registry we've created. If -we call upon the registry a second time, we get back the **same** :class:`.Session`:: - - >>> some_other_session = Session() - >>> some_session is some_other_session - True - -This pattern allows disparate sections of the application to call upon a global -:class:`.scoped_session`, so that all those areas may share the same session -without the need to pass it explicitly. The :class:`.Session` we've established -in our registry will remain, until we explicitly tell our registry to dispose of it, -by calling :meth:`.scoped_session.remove`:: - - >>> Session.remove() - -The :meth:`.scoped_session.remove` method first calls :meth:`.Session.close` on -the current :class:`.Session`, which has the effect of releasing any connection/transactional -resources owned by the :class:`.Session` first, then discarding the :class:`.Session` -itself. "Releasing" here means that connections are returned to their connection pool and any transactional state is rolled back, ultimately using the ``rollback()`` method of the underlying DBAPI connection. - -At this point, the :class:`.scoped_session` object is "empty", and will create -a **new** :class:`.Session` when called again. As illustrated below, this -is not the same :class:`.Session` we had before:: - - >>> new_session = Session() - >>> new_session is some_session - False - -The above series of steps illustrates the idea of the "registry" pattern in a -nutshell. With that basic idea in hand, we can discuss some of the details -of how this pattern proceeds. - -Implicit Method Access ----------------------- - -The job of the :class:`.scoped_session` is simple; hold onto a :class:`.Session` -for all who ask for it. As a means of producing more transparent access to this -:class:`.Session`, the :class:`.scoped_session` also includes **proxy behavior**, -meaning that the registry itself can be treated just like a :class:`.Session` -directly; when methods are called on this object, they are **proxied** to the -underlying :class:`.Session` being maintained by the registry:: - - Session = scoped_session(some_factory) - - # equivalent to: - # - # session = Session() - # print session.query(MyClass).all() - # - print Session.query(MyClass).all() - -The above code accomplishes the same task as that of acquiring the current -:class:`.Session` by calling upon the registry, then using that :class:`.Session`. - -Thread-Local Scope ------------------- - -Users who are familiar with multithreaded programming will note that representing -anything as a global variable is usually a bad idea, as it implies that the -global object will be accessed by many threads concurrently. The :class:`.Session` -object is entirely designed to be used in a **non-concurrent** fashion, which -in terms of multithreading means "only in one thread at a time". So our -above example of :class:`.scoped_session` usage, where the same :class:`.Session` -object is maintained across multiple calls, suggests that some process needs -to be in place such that mutltiple calls across many threads don't actually get -a handle to the same session. We call this notion **thread local storage**, -which means, a special object is used that will maintain a distinct object -per each application thread. Python provides this via the -`threading.local() `_ -construct. The :class:`.scoped_session` object by default uses this object -as storage, so that a single :class:`.Session` is maintained for all who call -upon the :class:`.scoped_session` registry, but only within the scope of a single -thread. Callers who call upon the registry in a different thread get a -:class:`.Session` instance that is local to that other thread. - -Using this technique, the :class:`.scoped_session` provides a quick and relatively -simple (if one is familiar with thread-local storage) way of providing -a single, global object in an application that is safe to be called upon -from multiple threads. - -The :meth:`.scoped_session.remove` method, as always, removes the current -:class:`.Session` associated with the thread, if any. However, one advantage of the -``threading.local()`` object is that if the application thread itself ends, the -"storage" for that thread is also garbage collected. So it is in fact "safe" to -use thread local scope with an application that spawns and tears down threads, -without the need to call :meth:`.scoped_session.remove`. However, the scope -of transactions themselves, i.e. ending them via :meth:`.Session.commit` or -:meth:`.Session.rollback`, will usually still be something that must be explicitly -arranged for at the appropriate time, unless the application actually ties the -lifespan of a thread to the lifespan of a transaction. - -.. _session_lifespan: - -Using Thread-Local Scope with Web Applications ----------------------------------------------- - -As discussed in the section :ref:`session_faq_whentocreate`, a web application -is architected around the concept of a **web request**, and integrating -such an application with the :class:`.Session` usually implies that the :class:`.Session` -will be associated with that request. As it turns out, most Python web frameworks, -with notable exceptions such as the asynchronous frameworks Twisted and -Tornado, use threads in a simple way, such that a particular web request is received, -processed, and completed within the scope of a single *worker thread*. When -the request ends, the worker thread is released to a pool of workers where it -is available to handle another request. - -This simple correspondence of web request and thread means that to associate a -:class:`.Session` with a thread implies it is also associated with the web request -running within that thread, and vice versa, provided that the :class:`.Session` is -created only after the web request begins and torn down just before the web request ends. -So it is a common practice to use :class:`.scoped_session` as a quick way -to integrate the :class:`.Session` with a web application. The sequence -diagram below illustrates this flow:: - - Web Server Web Framework SQLAlchemy ORM Code - -------------- -------------- ------------------------------ - startup -> Web framework # Session registry is established - initializes Session = scoped_session(sessionmaker()) - - incoming - web request -> web request -> # The registry is *optionally* - starts # called upon explicitly to create - # a Session local to the thread and/or request - Session() - - # the Session registry can otherwise - # be used at any time, creating the - # request-local Session() if not present, - # or returning the existing one - Session.query(MyClass) # ... - - Session.add(some_object) # ... - - # if data was modified, commit the - # transaction - Session.commit() - - web request ends -> # the registry is instructed to - # remove the Session - Session.remove() - - sends output <- - outgoing web <- - response - -Using the above flow, the process of integrating the :class:`.Session` with the -web application has exactly two requirements: - -1. Create a single :class:`.scoped_session` registry when the web application - first starts, ensuring that this object is accessible by the rest of the - application. -2. Ensure that :meth:`.scoped_session.remove` is called when the web request ends, - usually by integrating with the web framework's event system to establish - an "on request end" event. - -As noted earlier, the above pattern is **just one potential way** to integrate a :class:`.Session` -with a web framework, one which in particular makes the significant assumption -that the **web framework associates web requests with application threads**. It is -however **strongly recommended that the integration tools provided with the web framework -itself be used, if available**, instead of :class:`.scoped_session`. - -In particular, while using a thread local can be convenient, it is preferable that the :class:`.Session` be -associated **directly with the request**, rather than with -the current thread. The next section on custom scopes details a more advanced configuration -which can combine the usage of :class:`.scoped_session` with direct request based scope, or -any kind of scope. - -Using Custom Created Scopes ---------------------------- - -The :class:`.scoped_session` object's default behavior of "thread local" scope is only -one of many options on how to "scope" a :class:`.Session`. A custom scope can be defined -based on any existing system of getting at "the current thing we are working with". - -Suppose a web framework defines a library function ``get_current_request()``. An application -built using this framework can call this function at any time, and the result will be -some kind of ``Request`` object that represents the current request being processed. -If the ``Request`` object is hashable, then this function can be easily integrated with -:class:`.scoped_session` to associate the :class:`.Session` with the request. Below we illustrate -this in conjunction with a hypothetical event marker provided by the web framework -``on_request_end``, which allows code to be invoked whenever a request ends:: - - from my_web_framework import get_current_request, on_request_end - from sqlalchemy.orm import scoped_session, sessionmaker - - Session = scoped_session(sessionmaker(bind=some_engine), scopefunc=get_current_request) - - @on_request_end - def remove_session(req): - Session.remove() - -Above, we instantiate :class:`.scoped_session` in the usual way, except that we pass -our request-returning function as the "scopefunc". This instructs :class:`.scoped_session` -to use this function to generate a dictionary key whenever the registry is called upon -to return the current :class:`.Session`. In this case it is particularly important -that we ensure a reliable "remove" system is implemented, as this dictionary is not -otherwise self-managed. - - -Contextual Session API ----------------------- - -.. autoclass:: sqlalchemy.orm.scoping.scoped_session - :members: - -.. autoclass:: sqlalchemy.util.ScopedRegistry - :members: - -.. autoclass:: sqlalchemy.util.ThreadLocalRegistry - -.. _session_partitioning: - -Partitioning Strategies -======================= - -Simple Vertical Partitioning ----------------------------- - -Vertical partitioning places different kinds of objects, or different tables, -across multiple databases:: - - engine1 = create_engine('postgresql://db1') - engine2 = create_engine('postgresql://db2') - - Session = sessionmaker(twophase=True) - - # bind User operations to engine 1, Account operations to engine 2 - Session.configure(binds={User:engine1, Account:engine2}) - - session = Session() - -Above, operations against either class will make usage of the :class:`.Engine` -linked to that class. Upon a flush operation, similar rules take place -to ensure each class is written to the right database. - -The transactions among the multiple databases can optionally be coordinated -via two phase commit, if the underlying backend supports it. See -:ref:`session_twophase` for an example. - -Custom Vertical Partitioning ----------------------------- - -More comprehensive rule-based class-level partitioning can be built by -overriding the :meth:`.Session.get_bind` method. Below we illustrate -a custom :class:`.Session` which delivers the following rules: - -1. Flush operations are delivered to the engine named ``master``. - -2. Operations on objects that subclass ``MyOtherClass`` all - occur on the ``other`` engine. - -3. Read operations for all other classes occur on a random - choice of the ``slave1`` or ``slave2`` database. - -:: - - engines = { - 'master':create_engine("sqlite:///master.db"), - 'other':create_engine("sqlite:///other.db"), - 'slave1':create_engine("sqlite:///slave1.db"), - 'slave2':create_engine("sqlite:///slave2.db"), - } - - from sqlalchemy.orm import Session, sessionmaker - import random - - class RoutingSession(Session): - def get_bind(self, mapper=None, clause=None): - if mapper and issubclass(mapper.class_, MyOtherClass): - return engines['other'] - elif self._flushing: - return engines['master'] - else: - return engines[ - random.choice(['slave1','slave2']) - ] - -The above :class:`.Session` class is plugged in using the ``class_`` -argument to :class:`.sessionmaker`:: - - Session = sessionmaker(class_=RoutingSession) - -This approach can be combined with multiple :class:`.MetaData` objects, -using an approach such as that of using the declarative ``__abstract__`` -keyword, described at :ref:`declarative_abstract`. - -Horizontal Partitioning ------------------------ - -Horizontal partitioning partitions the rows of a single table (or a set of -tables) across multiple databases. - -See the "sharding" example: :ref:`examples_sharding`. - -.. _bulk_operations: - -Bulk Operations -=============== - -.. note:: Bulk Operations mode is a new series of operations made available - on the :class:`.Session` object for the purpose of invoking INSERT and - UPDATE statements with greatly reduced Python overhead, at the expense - of much less functionality, automation, and error checking. - As of SQLAlchemy 1.0, these features should be considered as "beta", and - additionally are intended for advanced users. - -.. versionadded:: 1.0.0 - -Bulk operations on the :class:`.Session` include :meth:`.Session.bulk_save_objects`, -:meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings`. -The purpose of these methods is to directly expose internal elements of the unit of work system, -such that facilities for emitting INSERT and UPDATE statements given dictionaries -or object states can be utilized alone, bypassing the normal unit of work -mechanics of state, relationship and attribute management. The advantages -to this approach is strictly one of reduced Python overhead: - -* The flush() process, including the survey of all objects, their state, - their cascade status, the status of all objects associated with them - via :func:`.relationship`, and the topological sort of all operations to - be performed is completely bypassed. This reduces a great amount of - Python overhead. - -* The objects as given have no defined relationship to the target - :class:`.Session`, even when the operation is complete, meaning there's no - overhead in attaching them or managing their state in terms of the identity - map or session. - -* The :meth:`.Session.bulk_insert_mappings` and :meth:`.Session.bulk_update_mappings` - methods accept lists of plain Python dictionaries, not objects; this further - reduces a large amount of overhead associated with instantiating mapped - objects and assigning state to them, which normally is also subject to - expensive tracking of history on a per-attribute basis. - -* The process of fetching primary keys after an INSERT also is disabled by - default. When performed correctly, INSERT statements can now more readily - be batched by the unit of work process into ``executemany()`` blocks, which - perform vastly better than individual statement invocations. - -* UPDATE statements can similarly be tailored such that all attributes - are subject to the SET clase unconditionally, again making it much more - likely that ``executemany()`` blocks can be used. - -The performance behavior of the bulk routines should be studied using the -:ref:`examples_performance` example suite. This is a series of example -scripts which illustrate Python call-counts across a variety of scenarios, -including bulk insert and update scenarios. - -.. seealso:: - - :ref:`examples_performance` - includes detailed examples of bulk operations - contrasted against traditional Core and ORM methods, including performance - metrics. - -Usage ------ - -The methods each work in the context of the :class:`.Session` object's -transaction, like any other:: - - s = Session() - objects = [ - User(name="u1"), - User(name="u2"), - User(name="u3") - ] - s.bulk_save_objects(objects) - -For :meth:`.Session.bulk_insert_mappings`, and :meth:`.Session.bulk_update_mappings`, -dictionaries are passed:: - - s.bulk_insert_mappings(User, - [dict(name="u1"), dict(name="u2"), dict(name="u3")] - ) - -.. seealso:: - - :meth:`.Session.bulk_save_objects` - - :meth:`.Session.bulk_insert_mappings` - - :meth:`.Session.bulk_update_mappings` - - -Comparison to Core Insert / Update Constructs ---------------------------------------------- - -The bulk methods offer performance that under particular circumstances -can be close to that of using the core :class:`.Insert` and -:class:`.Update` constructs in an "executemany" context (for a description -of "executemany", see :ref:`execute_multiple` in the Core tutorial). -In order to achieve this, the -:paramref:`.Session.bulk_insert_mappings.return_defaults` -flag should be disabled so that rows can be batched together. The example -suite in :ref:`examples_performance` should be carefully studied in order -to gain familiarity with how fast bulk performance can be achieved. - -ORM Compatibility ------------------ - -The bulk insert / update methods lose a significant amount of functionality -versus traditional ORM use. The following is a listing of features that -are **not available** when using these methods: - -* persistence along :func:`.relationship` linkages - -* sorting of rows within order of dependency; rows are inserted or updated - directly in the order in which they are passed to the methods - -* Session-management on the given objects, including attachment to the - session, identity map management. - -* Functionality related to primary key mutation, ON UPDATE cascade - -* SQL expression inserts / updates (e.g. :ref:`flush_embedded_sql_expressions`) - -* ORM events such as :meth:`.MapperEvents.before_insert`, etc. The bulk - session methods have no event support. - -Features that **are available** include: - -* INSERTs and UPDATEs of mapped objects - -* Version identifier support - -* Multi-table mappings, such as joined-inheritance - however, an object - to be inserted across multiple tables either needs to have primary key - identifiers fully populated ahead of time, else the - :paramref:`.Session.bulk_save_objects.return_defaults` flag must be used, - which will greatly reduce the performance benefits - - - - -Sessions API -============ - -Session and sessionmaker() ---------------------------- - -.. autoclass:: sessionmaker - :members: - :inherited-members: - -.. autoclass:: sqlalchemy.orm.session.Session - :members: - :inherited-members: - -.. autoclass:: sqlalchemy.orm.session.SessionTransaction - :members: - -Session Utilites ----------------- - -.. autofunction:: make_transient - -.. autofunction:: make_transient_to_detached - -.. autofunction:: object_session - -.. autofunction:: sqlalchemy.orm.util.was_deleted - -Attribute and State Management Utilities ------------------------------------------ - -These functions are provided by the SQLAlchemy attribute -instrumentation API to provide a detailed interface for dealing -with instances, attribute values, and history. Some of them -are useful when constructing event listener functions, such as -those described in :doc:`/orm/events`. - -.. currentmodule:: sqlalchemy.orm.util - -.. autofunction:: object_state - -.. currentmodule:: sqlalchemy.orm.attributes - -.. autofunction:: del_attribute - -.. autofunction:: get_attribute - -.. autofunction:: get_history - -.. autofunction:: init_collection - -.. autofunction:: flag_modified - -.. function:: instance_state - - Return the :class:`.InstanceState` for a given - mapped object. - - This function is the internal version - of :func:`.object_state`. The - :func:`.object_state` and/or the - :func:`.inspect` function is preferred here - as they each emit an informative exception - if the given object is not mapped. - -.. autofunction:: sqlalchemy.orm.instrumentation.is_instrumented - -.. autofunction:: set_attribute - -.. autofunction:: set_committed_value - -.. autoclass:: History - :members: diff --git a/doc/build/orm/session_api.rst b/doc/build/orm/session_api.rst new file mode 100644 index 000000000..64ac8c086 --- /dev/null +++ b/doc/build/orm/session_api.rst @@ -0,0 +1,74 @@ +Session API +============ + +Session and sessionmaker() +--------------------------- + +.. autoclass:: sessionmaker + :members: + :inherited-members: + +.. autoclass:: sqlalchemy.orm.session.Session + :members: + :inherited-members: + +.. autoclass:: sqlalchemy.orm.session.SessionTransaction + :members: + +Session Utilites +---------------- + +.. autofunction:: make_transient + +.. autofunction:: make_transient_to_detached + +.. autofunction:: object_session + +.. autofunction:: sqlalchemy.orm.util.was_deleted + +Attribute and State Management Utilities +----------------------------------------- + +These functions are provided by the SQLAlchemy attribute +instrumentation API to provide a detailed interface for dealing +with instances, attribute values, and history. Some of them +are useful when constructing event listener functions, such as +those described in :doc:`/orm/events`. + +.. currentmodule:: sqlalchemy.orm.util + +.. autofunction:: object_state + +.. currentmodule:: sqlalchemy.orm.attributes + +.. autofunction:: del_attribute + +.. autofunction:: get_attribute + +.. autofunction:: get_history + +.. autofunction:: init_collection + +.. autofunction:: flag_modified + +.. function:: instance_state + + Return the :class:`.InstanceState` for a given + mapped object. + + This function is the internal version + of :func:`.object_state`. The + :func:`.object_state` and/or the + :func:`.inspect` function is preferred here + as they each emit an informative exception + if the given object is not mapped. + +.. autofunction:: sqlalchemy.orm.instrumentation.is_instrumented + +.. autofunction:: set_attribute + +.. autofunction:: set_committed_value + +.. autoclass:: History + :members: + diff --git a/doc/build/orm/session_basics.rst b/doc/build/orm/session_basics.rst new file mode 100644 index 000000000..8919864ca --- /dev/null +++ b/doc/build/orm/session_basics.rst @@ -0,0 +1,744 @@ +========================== +Session Basics +========================== + +What does the Session do ? +========================== + +In the most general sense, the :class:`~.Session` establishes all +conversations with the database and represents a "holding zone" for all the +objects which you've loaded or associated with it during its lifespan. It +provides the entrypoint to acquire a :class:`.Query` object, which sends +queries to the database using the :class:`~.Session` object's current database +connection, populating result rows into objects that are then stored in the +:class:`.Session`, inside a structure called the `Identity Map +`_ - a data structure +that maintains unique copies of each object, where "unique" means "only one +object with a particular primary key". + +The :class:`.Session` begins in an essentially stateless form. Once queries +are issued or other objects are persisted with it, it requests a connection +resource from an :class:`.Engine` that is associated either with the +:class:`.Session` itself or with the mapped :class:`.Table` objects being +operated upon. This connection represents an ongoing transaction, which +remains in effect until the :class:`.Session` is instructed to commit or roll +back its pending state. + +All changes to objects maintained by a :class:`.Session` are tracked - before +the database is queried again or before the current transaction is committed, +it **flushes** all pending changes to the database. This is known as the `Unit +of Work `_ pattern. + +When using a :class:`.Session`, it's important to note that the objects +which are associated with it are **proxy objects** to the transaction being +held by the :class:`.Session` - there are a variety of events that will cause +objects to re-access the database in order to keep synchronized. It is +possible to "detach" objects from a :class:`.Session`, and to continue using +them, though this practice has its caveats. It's intended that +usually, you'd re-associate detached objects with another :class:`.Session` when you +want to work with them again, so that they can resume their normal task of +representing database state. + +.. _session_getting: + +Getting a Session +================= + +:class:`.Session` is a regular Python class which can +be directly instantiated. However, to standardize how sessions are configured +and acquired, the :class:`.sessionmaker` class is normally +used to create a top level :class:`.Session` +configuration which can then be used throughout an application without the +need to repeat the configurational arguments. + +The usage of :class:`.sessionmaker` is illustrated below: + +.. sourcecode:: python+sql + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + # an Engine, which the Session will use for connection + # resources + some_engine = create_engine('postgresql://scott:tiger@localhost/') + + # create a configured "Session" class + Session = sessionmaker(bind=some_engine) + + # create a Session + session = Session() + + # work with sess + myobject = MyObject('foo', 'bar') + session.add(myobject) + session.commit() + +Above, the :class:`.sessionmaker` call creates a factory for us, +which we assign to the name ``Session``. This factory, when +called, will create a new :class:`.Session` object using the configurational +arguments we've given the factory. In this case, as is typical, +we've configured the factory to specify a particular :class:`.Engine` for +connection resources. + +A typical setup will associate the :class:`.sessionmaker` with an :class:`.Engine`, +so that each :class:`.Session` generated will use this :class:`.Engine` +to acquire connection resources. This association can +be set up as in the example above, using the ``bind`` argument. + +When you write your application, place the +:class:`.sessionmaker` factory at the global level. This +factory can then +be used by the rest of the applcation as the source of new :class:`.Session` +instances, keeping the configuration for how :class:`.Session` objects +are constructed in one place. + +The :class:`.sessionmaker` factory can also be used in conjunction with +other helpers, which are passed a user-defined :class:`.sessionmaker` that +is then maintained by the helper. Some of these helpers are discussed in the +section :ref:`session_faq_whentocreate`. + +Adding Additional Configuration to an Existing sessionmaker() +-------------------------------------------------------------- + +A common scenario is where the :class:`.sessionmaker` is invoked +at module import time, however the generation of one or more :class:`.Engine` +instances to be associated with the :class:`.sessionmaker` has not yet proceeded. +For this use case, the :class:`.sessionmaker` construct offers the +:meth:`.sessionmaker.configure` method, which will place additional configuration +directives into an existing :class:`.sessionmaker` that will take place +when the construct is invoked:: + + + from sqlalchemy.orm import sessionmaker + from sqlalchemy import create_engine + + # configure Session class with desired options + Session = sessionmaker() + + # later, we create the engine + engine = create_engine('postgresql://...') + + # associate it with our custom Session class + Session.configure(bind=engine) + + # work with the session + session = Session() + +Creating Ad-Hoc Session Objects with Alternate Arguments +--------------------------------------------------------- + +For the use case where an application needs to create a new :class:`.Session` with +special arguments that deviate from what is normally used throughout the application, +such as a :class:`.Session` that binds to an alternate +source of connectivity, or a :class:`.Session` that should +have other arguments such as ``expire_on_commit`` established differently from +what most of the application wants, specific arguments can be passed to the +:class:`.sessionmaker` factory's :meth:`.sessionmaker.__call__` method. +These arguments will override whatever +configurations have already been placed, such as below, where a new :class:`.Session` +is constructed against a specific :class:`.Connection`:: + + # at the module level, the global sessionmaker, + # bound to a specific Engine + Session = sessionmaker(bind=engine) + + # later, some unit of code wants to create a + # Session that is bound to a specific Connection + conn = engine.connect() + session = Session(bind=conn) + +The typical rationale for the association of a :class:`.Session` with a specific +:class:`.Connection` is that of a test fixture that maintains an external +transaction - see :ref:`session_external_transaction` for an example of this. + + +.. _session_faq: + +Session Frequently Asked Questions +=================================== + +By this point, many users already have questions about sessions. +This section presents a mini-FAQ (note that we have also a `real FAQ `) +of the most basic issues one is presented with when using a :class:`.Session`. + +When do I make a :class:`.sessionmaker`? +------------------------------------------ + +Just one time, somewhere in your application's global scope. It should be +looked upon as part of your application's configuration. If your +application has three .py files in a package, you could, for example, +place the :class:`.sessionmaker` line in your ``__init__.py`` file; from +that point on your other modules say "from mypackage import Session". That +way, everyone else just uses :class:`.Session()`, +and the configuration of that session is controlled by that central point. + +If your application starts up, does imports, but does not know what +database it's going to be connecting to, you can bind the +:class:`.Session` at the "class" level to the +engine later on, using :meth:`.sessionmaker.configure`. + +In the examples in this section, we will frequently show the +:class:`.sessionmaker` being created right above the line where we actually +invoke :class:`.Session`. But that's just for +example's sake! In reality, the :class:`.sessionmaker` would be somewhere +at the module level. The calls to instantiate :class:`.Session` +would then be placed at the point in the application where database +conversations begin. + +.. _session_faq_whentocreate: + +When do I construct a :class:`.Session`, when do I commit it, and when do I close it? +------------------------------------------------------------------------------------- + +.. topic:: tl;dr; + + As a general rule, keep the lifecycle of the session **separate and + external** from functions and objects that access and/or manipulate + database data. + +A :class:`.Session` is typically constructed at the beginning of a logical +operation where database access is potentially anticipated. + +The :class:`.Session`, whenever it is used to talk to the database, +begins a database transaction as soon as it starts communicating. +Assuming the ``autocommit`` flag is left at its recommended default +of ``False``, this transaction remains in progress until the :class:`.Session` +is rolled back, committed, or closed. The :class:`.Session` will +begin a new transaction if it is used again, subsequent to the previous +transaction ending; from this it follows that the :class:`.Session` +is capable of having a lifespan across many transactions, though only +one at a time. We refer to these two concepts as **transaction scope** +and **session scope**. + +The implication here is that the SQLAlchemy ORM is encouraging the +developer to establish these two scopes in their application, +including not only when the scopes begin and end, but also the +expanse of those scopes, for example should a single +:class:`.Session` instance be local to the execution flow within a +function or method, should it be a global object used by the +entire application, or somewhere in between these two. + +The burden placed on the developer to determine this scope is one +area where the SQLAlchemy ORM necessarily has a strong opinion +about how the database should be used. The :term:`unit of work` pattern +is specifically one of accumulating changes over time and flushing +them periodically, keeping in-memory state in sync with what's +known to be present in a local transaction. This pattern is only +effective when meaningful transaction scopes are in place. + +It's usually not very hard to determine the best points at which +to begin and end the scope of a :class:`.Session`, though the wide +variety of application architectures possible can introduce +challenging situations. + +A common choice is to tear down the :class:`.Session` at the same +time the transaction ends, meaning the transaction and session scopes +are the same. This is a great choice to start out with as it +removes the need to consider session scope as separate from transaction +scope. + +While there's no one-size-fits-all recommendation for how transaction +scope should be determined, there are common patterns. Especially +if one is writing a web application, the choice is pretty much established. + +A web application is the easiest case because such an appication is already +constructed around a single, consistent scope - this is the **request**, +which represents an incoming request from a browser, the processing +of that request to formulate a response, and finally the delivery of that +response back to the client. Integrating web applications with the +:class:`.Session` is then the straightforward task of linking the +scope of the :class:`.Session` to that of the request. The :class:`.Session` +can be established as the request begins, or using a :term:`lazy initialization` +pattern which establishes one as soon as it is needed. The request +then proceeds, with some system in place where application logic can access +the current :class:`.Session` in a manner associated with how the actual +request object is accessed. As the request ends, the :class:`.Session` +is torn down as well, usually through the usage of event hooks provided +by the web framework. The transaction used by the :class:`.Session` +may also be committed at this point, or alternatively the application may +opt for an explicit commit pattern, only committing for those requests +where one is warranted, but still always tearing down the :class:`.Session` +unconditionally at the end. + +Some web frameworks include infrastructure to assist in the task +of aligning the lifespan of a :class:`.Session` with that of a web request. +This includes products such as `Flask-SQLAlchemy `_, +for usage in conjunction with the Flask web framework, +and `Zope-SQLAlchemy `_, +typically used with the Pyramid framework. +SQLAlchemy recommends that these products be used as available. + +In those situations where the integration libraries are not +provided or are insufficient, SQLAlchemy includes its own "helper" class known as +:class:`.scoped_session`. A tutorial on the usage of this object +is at :ref:`unitofwork_contextual`. It provides both a quick way +to associate a :class:`.Session` with the current thread, as well as +patterns to associate :class:`.Session` objects with other kinds of +scopes. + +As mentioned before, for non-web applications there is no one clear +pattern, as applications themselves don't have just one pattern +of architecture. The best strategy is to attempt to demarcate +"operations", points at which a particular thread begins to perform +a series of operations for some period of time, which can be committed +at the end. Some examples: + +* A background daemon which spawns off child forks + would want to create a :class:`.Session` local to each child + process, work with that :class:`.Session` through the life of the "job" + that the fork is handling, then tear it down when the job is completed. + +* For a command-line script, the application would create a single, global + :class:`.Session` that is established when the program begins to do its + work, and commits it right as the program is completing its task. + +* For a GUI interface-driven application, the scope of the :class:`.Session` + may best be within the scope of a user-generated event, such as a button + push. Or, the scope may correspond to explicit user interaction, such as + the user "opening" a series of records, then "saving" them. + +As a general rule, the application should manage the lifecycle of the +session *externally* to functions that deal with specific data. This is a +fundamental separation of concerns which keeps data-specific operations +agnostic of the context in which they access and manipulate that data. + +E.g. **don't do this**:: + + ### this is the **wrong way to do it** ### + + class ThingOne(object): + def go(self): + session = Session() + try: + session.query(FooBar).update({"x": 5}) + session.commit() + except: + session.rollback() + raise + + class ThingTwo(object): + def go(self): + session = Session() + try: + session.query(Widget).update({"q": 18}) + session.commit() + except: + session.rollback() + raise + + def run_my_program(): + ThingOne().go() + ThingTwo().go() + +Keep the lifecycle of the session (and usually the transaction) +**separate and external**:: + + ### this is a **better** (but not the only) way to do it ### + + class ThingOne(object): + def go(self, session): + session.query(FooBar).update({"x": 5}) + + class ThingTwo(object): + def go(self, session): + session.query(Widget).update({"q": 18}) + + def run_my_program(): + session = Session() + try: + ThingOne().go(session) + ThingTwo().go(session) + + session.commit() + except: + session.rollback() + raise + finally: + session.close() + +The advanced developer will try to keep the details of session, transaction +and exception management as far as possible from the details of the program +doing its work. For example, we can further separate concerns using a `context manager `_:: + + ### another way (but again *not the only way*) to do it ### + + from contextlib import contextmanager + + @contextmanager + def session_scope(): + """Provide a transactional scope around a series of operations.""" + session = Session() + try: + yield session + session.commit() + except: + session.rollback() + raise + finally: + session.close() + + + def run_my_program(): + with session_scope() as session: + ThingOne().go(session) + ThingTwo().go(session) + + +Is the Session a cache? +---------------------------------- + +Yeee...no. It's somewhat used as a cache, in that it implements the +:term:`identity map` pattern, and stores objects keyed to their primary key. +However, it doesn't do any kind of query caching. This means, if you say +``session.query(Foo).filter_by(name='bar')``, even if ``Foo(name='bar')`` +is right there, in the identity map, the session has no idea about that. +It has to issue SQL to the database, get the rows back, and then when it +sees the primary key in the row, *then* it can look in the local identity +map and see that the object is already there. It's only when you say +``query.get({some primary key})`` that the +:class:`~sqlalchemy.orm.session.Session` doesn't have to issue a query. + +Additionally, the Session stores object instances using a weak reference +by default. This also defeats the purpose of using the Session as a cache. + +The :class:`.Session` is not designed to be a +global object from which everyone consults as a "registry" of objects. +That's more the job of a **second level cache**. SQLAlchemy provides +a pattern for implementing second level caching using `dogpile.cache `_, +via the :ref:`examples_caching` example. + +How can I get the :class:`~sqlalchemy.orm.session.Session` for a certain object? +------------------------------------------------------------------------------------ + +Use the :meth:`~.Session.object_session` classmethod +available on :class:`~sqlalchemy.orm.session.Session`:: + + session = Session.object_session(someobject) + +The newer :ref:`core_inspection_toplevel` system can also be used:: + + from sqlalchemy import inspect + session = inspect(someobject).session + +.. _session_faq_threadsafe: + +Is the session thread-safe? +------------------------------ + +The :class:`.Session` is very much intended to be used in a +**non-concurrent** fashion, which usually means in only one thread at a +time. + +The :class:`.Session` should be used in such a way that one +instance exists for a single series of operations within a single +transaction. One expedient way to get this effect is by associating +a :class:`.Session` with the current thread (see :ref:`unitofwork_contextual` +for background). Another is to use a pattern +where the :class:`.Session` is passed between functions and is otherwise +not shared with other threads. + +The bigger point is that you should not *want* to use the session +with multiple concurrent threads. That would be like having everyone at a +restaurant all eat from the same plate. The session is a local "workspace" +that you use for a specific set of tasks; you don't want to, or need to, +share that session with other threads who are doing some other task. + +Making sure the :class:`.Session` is only used in a single concurrent thread at a time +is called a "share nothing" approach to concurrency. But actually, not +sharing the :class:`.Session` implies a more significant pattern; it +means not just the :class:`.Session` object itself, but +also **all objects that are associated with that Session**, must be kept within +the scope of a single concurrent thread. The set of mapped +objects associated with a :class:`.Session` are essentially proxies for data +within database rows accessed over a database connection, and so just like +the :class:`.Session` itself, the whole +set of objects is really just a large-scale proxy for a database connection +(or connections). Ultimately, it's mostly the DBAPI connection itself that +we're keeping away from concurrent access; but since the :class:`.Session` +and all the objects associated with it are all proxies for that DBAPI connection, +the entire graph is essentially not safe for concurrent access. + +If there are in fact multiple threads participating +in the same task, then you may consider sharing the session and its objects between +those threads; however, in this extremely unusual scenario the application would +need to ensure that a proper locking scheme is implemented so that there isn't +*concurrent* access to the :class:`.Session` or its state. A more common approach +to this situation is to maintain a single :class:`.Session` per concurrent thread, +but to instead *copy* objects from one :class:`.Session` to another, often +using the :meth:`.Session.merge` method to copy the state of an object into +a new object local to a different :class:`.Session`. + +Basics of Using a Session +=========================== + +The most basic :class:`.Session` use patterns are presented here. + +Querying +-------- + +The :meth:`~.Session.query` function takes one or more +*entities* and returns a new :class:`~sqlalchemy.orm.query.Query` object which +will issue mapper queries within the context of this Session. An entity is +defined as a mapped class, a :class:`~sqlalchemy.orm.mapper.Mapper` object, an +orm-enabled *descriptor*, or an ``AliasedClass`` object:: + + # query from a class + session.query(User).filter_by(name='ed').all() + + # query with multiple classes, returns tuples + session.query(User, Address).join('addresses').filter_by(name='ed').all() + + # query using orm-enabled descriptors + session.query(User.name, User.fullname).all() + + # query from a mapper + user_mapper = class_mapper(User) + session.query(user_mapper) + +When :class:`~sqlalchemy.orm.query.Query` returns results, each object +instantiated is stored within the identity map. When a row matches an object +which is already present, the same object is returned. In the latter case, +whether or not the row is populated onto an existing object depends upon +whether the attributes of the instance have been *expired* or not. A +default-configured :class:`~sqlalchemy.orm.session.Session` automatically +expires all instances along transaction boundaries, so that with a normally +isolated transaction, there shouldn't be any issue of instances representing +data which is stale with regards to the current transaction. + +The :class:`.Query` object is introduced in great detail in +:ref:`ormtutorial_toplevel`, and further documented in +:ref:`query_api_toplevel`. + +Adding New or Existing Items +---------------------------- + +:meth:`~.Session.add` is used to place instances in the +session. For *transient* (i.e. brand new) instances, this will have the effect +of an INSERT taking place for those instances upon the next flush. For +instances which are *persistent* (i.e. were loaded by this session), they are +already present and do not need to be added. Instances which are *detached* +(i.e. have been removed from a session) may be re-associated with a session +using this method:: + + user1 = User(name='user1') + user2 = User(name='user2') + session.add(user1) + session.add(user2) + + session.commit() # write changes to the database + +To add a list of items to the session at once, use +:meth:`~.Session.add_all`:: + + session.add_all([item1, item2, item3]) + +The :meth:`~.Session.add` operation **cascades** along +the ``save-update`` cascade. For more details see the section +:ref:`unitofwork_cascades`. + + +Deleting +-------- + +The :meth:`~.Session.delete` method places an instance +into the Session's list of objects to be marked as deleted:: + + # mark two objects to be deleted + session.delete(obj1) + session.delete(obj2) + + # commit (or flush) + session.commit() + +.. _session_deleting_from_collections: + +Deleting from Collections +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A common confusion that arises regarding :meth:`~.Session.delete` is when +objects which are members of a collection are being deleted. While the +collection member is marked for deletion from the database, this does not +impact the collection itself in memory until the collection is expired. +Below, we illustrate that even after an ``Address`` object is marked +for deletion, it's still present in the collection associated with the +parent ``User``, even after a flush:: + + >>> address = user.addresses[1] + >>> session.delete(address) + >>> session.flush() + >>> address in user.addresses + True + +When the above session is committed, all attributes are expired. The next +access of ``user.addresses`` will re-load the collection, revealing the +desired state:: + + >>> session.commit() + >>> address in user.addresses + False + +The usual practice of deleting items within collections is to forego the usage +of :meth:`~.Session.delete` directly, and instead use cascade behavior to +automatically invoke the deletion as a result of removing the object from +the parent collection. The ``delete-orphan`` cascade accomplishes this, +as illustrated in the example below:: + + mapper(User, users_table, properties={ + 'addresses':relationship(Address, cascade="all, delete, delete-orphan") + }) + del user.addresses[1] + session.flush() + +Where above, upon removing the ``Address`` object from the ``User.addresses`` +collection, the ``delete-orphan`` cascade has the effect of marking the ``Address`` +object for deletion in the same way as passing it to :meth:`~.Session.delete`. + +See also :ref:`unitofwork_cascades` for detail on cascades. + +Deleting based on Filter Criterion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The caveat with ``Session.delete()`` is that you need to have an object handy +already in order to delete. The Query includes a +:func:`~sqlalchemy.orm.query.Query.delete` method which deletes based on +filtering criteria:: + + session.query(User).filter(User.id==7).delete() + +The ``Query.delete()`` method includes functionality to "expire" objects +already in the session which match the criteria. However it does have some +caveats, including that "delete" and "delete-orphan" cascades won't be fully +expressed for collections which are already loaded. See the API docs for +:meth:`~sqlalchemy.orm.query.Query.delete` for more details. + +.. _session_flushing: + +Flushing +-------- + +When the :class:`~sqlalchemy.orm.session.Session` is used with its default +configuration, the flush step is nearly always done transparently. +Specifically, the flush occurs before any individual +:class:`~sqlalchemy.orm.query.Query` is issued, as well as within the +:meth:`~.Session.commit` call before the transaction is +committed. It also occurs before a SAVEPOINT is issued when +:meth:`~.Session.begin_nested` is used. + +Regardless of the autoflush setting, a flush can always be forced by issuing +:meth:`~.Session.flush`:: + + session.flush() + +The "flush-on-Query" aspect of the behavior can be disabled by constructing +:class:`.sessionmaker` with the flag ``autoflush=False``:: + + Session = sessionmaker(autoflush=False) + +Additionally, autoflush can be temporarily disabled by setting the +``autoflush`` flag at any time:: + + mysession = Session() + mysession.autoflush = False + +Some autoflush-disable recipes are available at `DisableAutoFlush +`_. + +The flush process *always* occurs within a transaction, even if the +:class:`~sqlalchemy.orm.session.Session` has been configured with +``autocommit=True``, a setting that disables the session's persistent +transactional state. If no transaction is present, +:meth:`~.Session.flush` creates its own transaction and +commits it. Any failures during flush will always result in a rollback of +whatever transaction is present. If the Session is not in ``autocommit=True`` +mode, an explicit call to :meth:`~.Session.rollback` is +required after a flush fails, even though the underlying transaction will have +been rolled back already - this is so that the overall nesting pattern of +so-called "subtransactions" is consistently maintained. + +.. _session_committing: + +Committing +---------- + +:meth:`~.Session.commit` is used to commit the current +transaction. It always issues :meth:`~.Session.flush` +beforehand to flush any remaining state to the database; this is independent +of the "autoflush" setting. If no transaction is present, it raises an error. +Note that the default behavior of the :class:`~sqlalchemy.orm.session.Session` +is that a "transaction" is always present; this behavior can be disabled by +setting ``autocommit=True``. In autocommit mode, a transaction can be +initiated by calling the :meth:`~.Session.begin` method. + +.. note:: + + The term "transaction" here refers to a transactional + construct within the :class:`.Session` itself which may be + maintaining zero or more actual database (DBAPI) transactions. An individual + DBAPI connection begins participation in the "transaction" as it is first + used to execute a SQL statement, then remains present until the session-level + "transaction" is completed. See :ref:`unitofwork_transaction` for + further detail. + +Another behavior of :meth:`~.Session.commit` is that by +default it expires the state of all instances present after the commit is +complete. This is so that when the instances are next accessed, either through +attribute access or by them being present in a +:class:`~sqlalchemy.orm.query.Query` result set, they receive the most recent +state. To disable this behavior, configure +:class:`.sessionmaker` with ``expire_on_commit=False``. + +Normally, instances loaded into the :class:`~sqlalchemy.orm.session.Session` +are never changed by subsequent queries; the assumption is that the current +transaction is isolated so the state most recently loaded is correct as long +as the transaction continues. Setting ``autocommit=True`` works against this +model to some degree since the :class:`~sqlalchemy.orm.session.Session` +behaves in exactly the same way with regard to attribute state, except no +transaction is present. + +.. _session_rollback: + +Rolling Back +------------ + +:meth:`~.Session.rollback` rolls back the current +transaction. With a default configured session, the post-rollback state of the +session is as follows: + + * All transactions are rolled back and all connections returned to the + connection pool, unless the Session was bound directly to a Connection, in + which case the connection is still maintained (but still rolled back). + * Objects which were initially in the *pending* state when they were added + to the :class:`~sqlalchemy.orm.session.Session` within the lifespan of the + transaction are expunged, corresponding to their INSERT statement being + rolled back. The state of their attributes remains unchanged. + * Objects which were marked as *deleted* within the lifespan of the + transaction are promoted back to the *persistent* state, corresponding to + their DELETE statement being rolled back. Note that if those objects were + first *pending* within the transaction, that operation takes precedence + instead. + * All objects not expunged are fully expired. + +With that state understood, the :class:`~sqlalchemy.orm.session.Session` may +safely continue usage after a rollback occurs. + +When a :meth:`~.Session.flush` fails, typically for +reasons like primary key, foreign key, or "not nullable" constraint +violations, a :meth:`~.Session.rollback` is issued +automatically (it's currently not possible for a flush to continue after a +partial failure). However, the flush process always uses its own transactional +demarcator called a *subtransaction*, which is described more fully in the +docstrings for :class:`~sqlalchemy.orm.session.Session`. What it means here is +that even though the database transaction has been rolled back, the end user +must still issue :meth:`~.Session.rollback` to fully +reset the state of the :class:`~sqlalchemy.orm.session.Session`. + + +Closing +------- + +The :meth:`~.Session.close` method issues a +:meth:`~.Session.expunge_all`, and :term:`releases` any +transactional/connection resources. When connections are returned to the +connection pool, transactional state is rolled back as well. + + diff --git a/doc/build/orm/session_state_management.rst b/doc/build/orm/session_state_management.rst new file mode 100644 index 000000000..1ca7ca2e4 --- /dev/null +++ b/doc/build/orm/session_state_management.rst @@ -0,0 +1,560 @@ +State Management +================ + +.. _session_object_states: + +Quickie Intro to Object States +------------------------------ + +It's helpful to know the states which an instance can have within a session: + +* **Transient** - an instance that's not in a session, and is not saved to the + database; i.e. it has no database identity. The only relationship such an + object has to the ORM is that its class has a ``mapper()`` associated with + it. + +* **Pending** - when you :meth:`~.Session.add` a transient + instance, it becomes pending. It still wasn't actually flushed to the + database yet, but it will be when the next flush occurs. + +* **Persistent** - An instance which is present in the session and has a record + in the database. You get persistent instances by either flushing so that the + pending instances become persistent, or by querying the database for + existing instances (or moving persistent instances from other sessions into + your local session). + +* **Detached** - an instance which has a record in the database, but is not in + any session. There's nothing wrong with this, and you can use objects + normally when they're detached, **except** they will not be able to issue + any SQL in order to load collections or attributes which are not yet loaded, + or were marked as "expired". + +Knowing these states is important, since the +:class:`.Session` tries to be strict about ambiguous +operations (such as trying to save the same object to two different sessions +at the same time). + +Getting the Current State of an Object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The actual state of any mapped object can be viewed at any time using +the :func:`.inspect` system:: + + >>> from sqlalchemy import inspect + >>> insp = inspect(my_object) + >>> insp.persistent + True + +.. seealso:: + + :attr:`.InstanceState.transient` + + :attr:`.InstanceState.pending` + + :attr:`.InstanceState.persistent` + + :attr:`.InstanceState.detached` + + +Session Attributes +------------------ + +The :class:`~sqlalchemy.orm.session.Session` itself acts somewhat like a +set-like collection. All items present may be accessed using the iterator +interface:: + + for obj in session: + print obj + +And presence may be tested for using regular "contains" semantics:: + + if obj in session: + print "Object is present" + +The session is also keeping track of all newly created (i.e. pending) objects, +all objects which have had changes since they were last loaded or saved (i.e. +"dirty"), and everything that's been marked as deleted:: + + # pending objects recently added to the Session + session.new + + # persistent objects which currently have changes detected + # (this collection is now created on the fly each time the property is called) + session.dirty + + # persistent objects that have been marked as deleted via session.delete(obj) + session.deleted + + # dictionary of all persistent objects, keyed on their + # identity key + session.identity_map + +(Documentation: :attr:`.Session.new`, :attr:`.Session.dirty`, +:attr:`.Session.deleted`, :attr:`.Session.identity_map`). + +Note that objects within the session are by default *weakly referenced*. This +means that when they are dereferenced in the outside application, they fall +out of scope from within the :class:`~sqlalchemy.orm.session.Session` as well +and are subject to garbage collection by the Python interpreter. The +exceptions to this include objects which are pending, objects which are marked +as deleted, or persistent objects which have pending changes on them. After a +full flush, these collections are all empty, and all objects are again weakly +referenced. To disable the weak referencing behavior and force all objects +within the session to remain until explicitly expunged, configure +:class:`.sessionmaker` with the ``weak_identity_map=False`` +setting. + +.. _unitofwork_merging: + +Merging +------- + +:meth:`~.Session.merge` transfers state from an +outside object into a new or already existing instance within a session. It +also reconciles the incoming data against the state of the +database, producing a history stream which will be applied towards the next +flush, or alternatively can be made to produce a simple "transfer" of +state without producing change history or accessing the database. Usage is as follows:: + + merged_object = session.merge(existing_object) + +When given an instance, it follows these steps: + +* It examines the primary key of the instance. If it's present, it attempts + to locate that instance in the local identity map. If the ``load=True`` + flag is left at its default, it also checks the database for this primary + key if not located locally. +* If the given instance has no primary key, or if no instance can be found + with the primary key given, a new instance is created. +* The state of the given instance is then copied onto the located/newly + created instance. For attributes which are present on the source + instance, the value is transferred to the target instance. For mapped + attributes which aren't present on the source, the attribute is + expired on the target instance, discarding its existing value. + + If the ``load=True`` flag is left at its default, + this copy process emits events and will load the target object's + unloaded collections for each attribute present on the source object, + so that the incoming state can be reconciled against what's + present in the database. If ``load`` + is passed as ``False``, the incoming data is "stamped" directly without + producing any history. +* The operation is cascaded to related objects and collections, as + indicated by the ``merge`` cascade (see :ref:`unitofwork_cascades`). +* The new instance is returned. + +With :meth:`~.Session.merge`, the given "source" +instance is not modified nor is it associated with the target :class:`.Session`, +and remains available to be merged with any number of other :class:`.Session` +objects. :meth:`~.Session.merge` is useful for +taking the state of any kind of object structure without regard for its +origins or current session associations and copying its state into a +new session. Here's some examples: + +* An application which reads an object structure from a file and wishes to + save it to the database might parse the file, build up the + structure, and then use + :meth:`~.Session.merge` to save it + to the database, ensuring that the data within the file is + used to formulate the primary key of each element of the + structure. Later, when the file has changed, the same + process can be re-run, producing a slightly different + object structure, which can then be ``merged`` in again, + and the :class:`~sqlalchemy.orm.session.Session` will + automatically update the database to reflect those + changes, loading each object from the database by primary key and + then updating its state with the new state given. + +* An application is storing objects in an in-memory cache, shared by + many :class:`.Session` objects simultaneously. :meth:`~.Session.merge` + is used each time an object is retrieved from the cache to create + a local copy of it in each :class:`.Session` which requests it. + The cached object remains detached; only its state is moved into + copies of itself that are local to individual :class:`~.Session` + objects. + + In the caching use case, it's common to use the ``load=False`` + flag to remove the overhead of reconciling the object's state + with the database. There's also a "bulk" version of + :meth:`~.Session.merge` called :meth:`~.Query.merge_result` + that was designed to work with cache-extended :class:`.Query` + objects - see the section :ref:`examples_caching`. + +* An application wants to transfer the state of a series of objects + into a :class:`.Session` maintained by a worker thread or other + concurrent system. :meth:`~.Session.merge` makes a copy of each object + to be placed into this new :class:`.Session`. At the end of the operation, + the parent thread/process maintains the objects it started with, + and the thread/worker can proceed with local copies of those objects. + + In the "transfer between threads/processes" use case, the application + may want to use the ``load=False`` flag as well to avoid overhead and + redundant SQL queries as the data is transferred. + +Merge Tips +~~~~~~~~~~ + +:meth:`~.Session.merge` is an extremely useful method for many purposes. However, +it deals with the intricate border between objects that are transient/detached and +those that are persistent, as well as the automated transference of state. +The wide variety of scenarios that can present themselves here often require a +more careful approach to the state of objects. Common problems with merge usually involve +some unexpected state regarding the object being passed to :meth:`~.Session.merge`. + +Lets use the canonical example of the User and Address objects:: + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + name = Column(String(50), nullable=False) + addresses = relationship("Address", backref="user") + + class Address(Base): + __tablename__ = 'address' + + id = Column(Integer, primary_key=True) + email_address = Column(String(50), nullable=False) + user_id = Column(Integer, ForeignKey('user.id'), nullable=False) + +Assume a ``User`` object with one ``Address``, already persistent:: + + >>> u1 = User(name='ed', addresses=[Address(email_address='ed@ed.com')]) + >>> session.add(u1) + >>> session.commit() + +We now create ``a1``, an object outside the session, which we'd like +to merge on top of the existing ``Address``:: + + >>> existing_a1 = u1.addresses[0] + >>> a1 = Address(id=existing_a1.id) + +A surprise would occur if we said this:: + + >>> a1.user = u1 + >>> a1 = session.merge(a1) + >>> session.commit() + sqlalchemy.orm.exc.FlushError: New instance
+ with identity key (, (1,)) conflicts with + persistent instance
+ +Why is that ? We weren't careful with our cascades. The assignment +of ``a1.user`` to a persistent object cascaded to the backref of ``User.addresses`` +and made our ``a1`` object pending, as though we had added it. Now we have +*two* ``Address`` objects in the session:: + + >>> a1 = Address() + >>> a1.user = u1 + >>> a1 in session + True + >>> existing_a1 in session + True + >>> a1 is existing_a1 + False + +Above, our ``a1`` is already pending in the session. The +subsequent :meth:`~.Session.merge` operation essentially +does nothing. Cascade can be configured via the :paramref:`~.relationship.cascade` +option on :func:`.relationship`, although in this case it +would mean removing the ``save-update`` cascade from the +``User.addresses`` relationship - and usually, that behavior +is extremely convenient. The solution here would usually be to not assign +``a1.user`` to an object already persistent in the target +session. + +The ``cascade_backrefs=False`` option of :func:`.relationship` +will also prevent the ``Address`` from +being added to the session via the ``a1.user = u1`` assignment. + +Further detail on cascade operation is at :ref:`unitofwork_cascades`. + +Another example of unexpected state:: + + >>> a1 = Address(id=existing_a1.id, user_id=u1.id) + >>> assert a1.user is None + >>> True + >>> a1 = session.merge(a1) + >>> session.commit() + sqlalchemy.exc.IntegrityError: (IntegrityError) address.user_id + may not be NULL + +Here, we accessed a1.user, which returned its default value +of ``None``, which as a result of this access, has been placed in the ``__dict__`` of +our object ``a1``. Normally, this operation creates no change event, +so the ``user_id`` attribute takes precedence during a +flush. But when we merge the ``Address`` object into the session, the operation +is equivalent to:: + + >>> existing_a1.id = existing_a1.id + >>> existing_a1.user_id = u1.id + >>> existing_a1.user = None + +Where above, both ``user_id`` and ``user`` are assigned to, and change events +are emitted for both. The ``user`` association +takes precedence, and None is applied to ``user_id``, causing a failure. + +Most :meth:`~.Session.merge` issues can be examined by first checking - +is the object prematurely in the session ? + +.. sourcecode:: python+sql + + >>> a1 = Address(id=existing_a1, user_id=user.id) + >>> assert a1 not in session + >>> a1 = session.merge(a1) + +Or is there state on the object that we don't want ? Examining ``__dict__`` +is a quick way to check:: + + >>> a1 = Address(id=existing_a1, user_id=user.id) + >>> a1.user + >>> a1.__dict__ + {'_sa_instance_state': , + 'user_id': 1, + 'id': 1, + 'user': None} + >>> # we don't want user=None merged, remove it + >>> del a1.user + >>> a1 = session.merge(a1) + >>> # success + >>> session.commit() + +Expunging +--------- + +Expunge removes an object from the Session, sending persistent instances to +the detached state, and pending instances to the transient state: + +.. sourcecode:: python+sql + + session.expunge(obj1) + +To remove all items, call :meth:`~.Session.expunge_all` +(this method was formerly known as ``clear()``). + +.. _session_expire: + +Refreshing / Expiring +--------------------- + +:term:`Expiring` means that the database-persisted data held inside a series +of object attributes is erased, in such a way that when those attributes +are next accessed, a SQL query is emitted which will refresh that data from +the database. + +When we talk about expiration of data we are usually talking about an object +that is in the :term:`persistent` state. For example, if we load an object +as follows:: + + user = session.query(User).filter_by(name='user1').first() + +The above ``User`` object is persistent, and has a series of attributes +present; if we were to look inside its ``__dict__``, we'd see that state +loaded:: + + >>> user.__dict__ + { + 'id': 1, 'name': u'user1', + '_sa_instance_state': <...>, + } + +where ``id`` and ``name`` refer to those columns in the database. +``_sa_instance_state`` is a non-database-persisted value used by SQLAlchemy +internally (it refers to the :class:`.InstanceState` for the instance. +While not directly relevant to this section, if we want to get at it, +we should use the :func:`.inspect` function to access it). + +At this point, the state in our ``User`` object matches that of the loaded +database row. But upon expiring the object using a method such as +:meth:`.Session.expire`, we see that the state is removed:: + + >>> session.expire(user) + >>> user.__dict__ + {'_sa_instance_state': <...>} + +We see that while the internal "state" still hangs around, the values which +correspond to the ``id`` and ``name`` columns are gone. If we were to access +one of these columns and are watching SQL, we'd see this: + +.. sourcecode:: python+sql + + >>> print(user.name) + {opensql}SELECT user.id AS user_id, user.name AS user_name + FROM user + WHERE user.id = ? + (1,) + {stop}user1 + +Above, upon accessing the expired attribute ``user.name``, the ORM initiated +a :term:`lazy load` to retrieve the most recent state from the database, +by emitting a SELECT for the user row to which this user refers. Afterwards, +the ``__dict__`` is again populated:: + + >>> user.__dict__ + { + 'id': 1, 'name': u'user1', + '_sa_instance_state': <...>, + } + +.. note:: While we are peeking inside of ``__dict__`` in order to see a bit + of what SQLAlchemy does with object attributes, we **should not modify** + the contents of ``__dict__`` directly, at least as far as those attributes + which the SQLAlchemy ORM is maintaining (other attributes outside of SQLA's + realm are fine). This is because SQLAlchemy uses :term:`descriptors` in + order to track the changes we make to an object, and when we modify ``__dict__`` + directly, the ORM won't be able to track that we changed something. + +Another key behavior of both :meth:`~.Session.expire` and :meth:`~.Session.refresh` +is that all un-flushed changes on an object are discarded. That is, +if we were to modify an attribute on our ``User``:: + + >>> user.name = 'user2' + +but then we call :meth:`~.Session.expire` without first calling :meth:`~.Session.flush`, +our pending value of ``'user2'`` is discarded:: + + >>> session.expire(user) + >>> user.name + 'user1' + +The :meth:`~.Session.expire` method can be used to mark as "expired" all ORM-mapped +attributes for an instance:: + + # expire all ORM-mapped attributes on obj1 + session.expire(obj1) + +it can also be passed a list of string attribute names, referring to specific +attributes to be marked as expired:: + + # expire only attributes obj1.attr1, obj1.attr2 + session.expire(obj1, ['attr1', 'attr2']) + +The :meth:`~.Session.refresh` method has a similar interface, but instead +of expiring, it emits an immediate SELECT for the object's row immediately:: + + # reload all attributes on obj1 + session.refresh(obj1) + +:meth:`~.Session.refresh` also accepts a list of string attribute names, +but unlike :meth:`~.Session.expire`, expects at least one name to +be that of a column-mapped attribute:: + + # reload obj1.attr1, obj1.attr2 + session.refresh(obj1, ['attr1', 'attr2']) + +The :meth:`.Session.expire_all` method allows us to essentially call +:meth:`.Session.expire` on all objects contained within the :class:`.Session` +at once:: + + session.expire_all() + +What Actually Loads +~~~~~~~~~~~~~~~~~~~ + +The SELECT statement that's emitted when an object marked with :meth:`~.Session.expire` +or loaded with :meth:`~.Session.refresh` varies based on several factors, including: + +* The load of expired attributes is triggered from **column-mapped attributes only**. + While any kind of attribute can be marked as expired, including a + :func:`.relationship` - mapped attribute, accessing an expired :func:`.relationship` + attribute will emit a load only for that attribute, using standard + relationship-oriented lazy loading. Column-oriented attributes, even if + expired, will not load as part of this operation, and instead will load when + any column-oriented attribute is accessed. + +* :func:`.relationship`- mapped attributes will not load in response to + expired column-based attributes being accessed. + +* Regarding relationships, :meth:`~.Session.refresh` is more restrictive than + :meth:`~.Session.expire` with regards to attributes that aren't column-mapped. + Calling :meth:`.refresh` and passing a list of names that only includes + relationship-mapped attributes will actually raise an error. + In any case, non-eager-loading :func:`.relationship` attributes will not be + included in any refresh operation. + +* :func:`.relationship` attributes configured as "eager loading" via the + :paramref:`~.relationship.lazy` parameter will load in the case of + :meth:`~.Session.refresh`, if either no attribute names are specified, or + if their names are inclued in the list of attributes to be + refreshed. + +* Attributes that are configured as :func:`.deferred` will not normally load, + during either the expired-attribute load or during a refresh. + An unloaded attribute that's :func:`.deferred` instead loads on its own when directly + accessed, or if part of a "group" of deferred attributes where an unloaded + attribute in that group is accessed. + +* For expired attributes that are loaded on access, a joined-inheritance table + mapping will emit a SELECT that typically only includes those tables for which + unloaded attributes are present. The action here is sophisticated enough + to load only the parent or child table, for example, if the subset of columns + that were originally expired encompass only one or the other of those tables. + +* When :meth:`~.Session.refresh` is used on a joined-inheritance table mapping, + the SELECT emitted will resemble that of when :meth:`.Session.query` is + used on the target object's class. This is typically all those tables that + are set up as part of the mapping. + + +When to Expire or Refresh +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :class:`.Session` uses the expiration feature automatically whenever +the transaction referred to by the session ends. Meaning, whenever :meth:`.Session.commit` +or :meth:`.Session.rollback` is called, all objects within the :class:`.Session` +are expired, using a feature equivalent to that of the :meth:`.Session.expire_all` +method. The rationale is that the end of a transaction is a +demarcating point at which there is no more context available in order to know +what the current state of the database is, as any number of other transactions +may be affecting it. Only when a new transaction starts can we again have access +to the current state of the database, at which point any number of changes +may have occurred. + +.. sidebar:: Transaction Isolation + + Of course, most databases are capable of handling + multiple transactions at once, even involving the same rows of data. When + a relational database handles multiple transactions involving the same + tables or rows, this is when the :term:`isolation` aspect of the database comes + into play. The isolation behavior of different databases varies considerably + and even on a single database can be configured to behave in different ways + (via the so-called :term:`isolation level` setting). In that sense, the :class:`.Session` + can't fully predict when the same SELECT statement, emitted a second time, + will definitely return the data we already have, or will return new data. + So as a best guess, it assumes that within the scope of a transaction, unless + it is known that a SQL expression has been emitted to modify a particular row, + there's no need to refresh a row unless explicitly told to do so. + +The :meth:`.Session.expire` and :meth:`.Session.refresh` methods are used in +those cases when one wants to force an object to re-load its data from the +database, in those cases when it is known that the current state of data +is possibly stale. Reasons for this might include: + +* some SQL has been emitted within the transaction outside of the + scope of the ORM's object handling, such as if a :meth:`.Table.update` construct + were emitted using the :meth:`.Session.execute` method; + +* if the application + is attempting to acquire data that is known to have been modified in a + concurrent transaction, and it is also known that the isolation rules in effect + allow this data to be visible. + +The second bullet has the important caveat that "it is also known that the isolation rules in effect +allow this data to be visible." This means that it cannot be assumed that an +UPDATE that happened on another database connection will yet be visible here +locally; in many cases, it will not. This is why if one wishes to use +:meth:`.expire` or :meth:`.refresh` in order to view data between ongoing +transactions, an understanding of the isolation behavior in effect is essential. + +.. seealso:: + + :meth:`.Session.expire` + + :meth:`.Session.expire_all` + + :meth:`.Session.refresh` + + :term:`isolation` - glossary explanation of isolation which includes links + to Wikipedia. + + `The SQLAlchemy Session In-Depth `_ - a video + slides with an in-depth discussion of the object + lifecycle including the role of data expiration. diff --git a/doc/build/orm/session_transaction.rst b/doc/build/orm/session_transaction.rst new file mode 100644 index 000000000..ce5757dd0 --- /dev/null +++ b/doc/build/orm/session_transaction.rst @@ -0,0 +1,365 @@ +======================================= +Transactions and Connection Management +======================================= + +.. _unitofwork_transaction: + +Managing Transactions +===================== + +A newly constructed :class:`.Session` may be said to be in the "begin" state. +In this state, the :class:`.Session` has not established any connection or +transactional state with any of the :class:`.Engine` objects that may be associated +with it. + +The :class:`.Session` then receives requests to operate upon a database connection. +Typically, this means it is called upon to execute SQL statements using a particular +:class:`.Engine`, which may be via :meth:`.Session.query`, :meth:`.Session.execute`, +or within a flush operation of pending data, which occurs when such state exists +and :meth:`.Session.commit` or :meth:`.Session.flush` is called. + +As these requests are received, each new :class:`.Engine` encountered is associated +with an ongoing transactional state maintained by the :class:`.Session`. +When the first :class:`.Engine` is operated upon, the :class:`.Session` can be said +to have left the "begin" state and entered "transactional" state. For each +:class:`.Engine` encountered, a :class:`.Connection` is associated with it, +which is acquired via the :meth:`.Engine.contextual_connect` method. If a +:class:`.Connection` was directly associated with the :class:`.Session` (see :ref:`session_external_transaction` +for an example of this), it is +added to the transactional state directly. + +For each :class:`.Connection`, the :class:`.Session` also maintains a :class:`.Transaction` object, +which is acquired by calling :meth:`.Connection.begin` on each :class:`.Connection`, +or if the :class:`.Session` +object has been established using the flag ``twophase=True``, a :class:`.TwoPhaseTransaction` +object acquired via :meth:`.Connection.begin_twophase`. These transactions are all committed or +rolled back corresponding to the invocation of the +:meth:`.Session.commit` and :meth:`.Session.rollback` methods. A commit operation will +also call the :meth:`.TwoPhaseTransaction.prepare` method on all transactions if applicable. + +When the transactional state is completed after a rollback or commit, the :class:`.Session` +:term:`releases` all :class:`.Transaction` and :class:`.Connection` resources, +and goes back to the "begin" state, which +will again invoke new :class:`.Connection` and :class:`.Transaction` objects as new +requests to emit SQL statements are received. + +The example below illustrates this lifecycle:: + + engine = create_engine("...") + Session = sessionmaker(bind=engine) + + # new session. no connections are in use. + session = Session() + try: + # first query. a Connection is acquired + # from the Engine, and a Transaction + # started. + item1 = session.query(Item).get(1) + + # second query. the same Connection/Transaction + # are used. + item2 = session.query(Item).get(2) + + # pending changes are created. + item1.foo = 'bar' + item2.bar = 'foo' + + # commit. The pending changes above + # are flushed via flush(), the Transaction + # is committed, the Connection object closed + # and discarded, the underlying DBAPI connection + # returned to the connection pool. + session.commit() + except: + # on rollback, the same closure of state + # as that of commit proceeds. + session.rollback() + raise + +.. _session_begin_nested: + +Using SAVEPOINT +--------------- + +SAVEPOINT transactions, if supported by the underlying engine, may be +delineated using the :meth:`~.Session.begin_nested` +method:: + + Session = sessionmaker() + session = Session() + session.add(u1) + session.add(u2) + + session.begin_nested() # establish a savepoint + session.add(u3) + session.rollback() # rolls back u3, keeps u1 and u2 + + session.commit() # commits u1 and u2 + +:meth:`~.Session.begin_nested` may be called any number +of times, which will issue a new SAVEPOINT with a unique identifier for each +call. For each :meth:`~.Session.begin_nested` call, a +corresponding :meth:`~.Session.rollback` or +:meth:`~.Session.commit` must be issued. (But note that if the return value is +used as a context manager, i.e. in a with-statement, then this rollback/commit +is issued by the context manager upon exiting the context, and so should not be +added explicitly.) + +When :meth:`~.Session.begin_nested` is called, a +:meth:`~.Session.flush` is unconditionally issued +(regardless of the ``autoflush`` setting). This is so that when a +:meth:`~.Session.rollback` occurs, the full state of the +session is expired, thus causing all subsequent attribute/instance access to +reference the full state of the :class:`~sqlalchemy.orm.session.Session` right +before :meth:`~.Session.begin_nested` was called. + +:meth:`~.Session.begin_nested`, in the same manner as the less often +used :meth:`~.Session.begin` method, returns a transactional object +which also works as a context manager. +It can be succinctly used around individual record inserts in order to catch +things like unique constraint exceptions:: + + for record in records: + try: + with session.begin_nested(): + session.merge(record) + except: + print "Skipped record %s" % record + session.commit() + +.. _session_autocommit: + +Autocommit Mode +--------------- + +The example of :class:`.Session` transaction lifecycle illustrated at +the start of :ref:`unitofwork_transaction` applies to a :class:`.Session` configured in the +default mode of ``autocommit=False``. Constructing a :class:`.Session` +with ``autocommit=True`` produces a :class:`.Session` placed into "autocommit" mode, where each SQL statement +invoked by a :meth:`.Session.query` or :meth:`.Session.execute` occurs +using a new connection from the connection pool, discarding it after +results have been iterated. The :meth:`.Session.flush` operation +still occurs within the scope of a single transaction, though this transaction +is closed out after the :meth:`.Session.flush` operation completes. + +.. warning:: + + "autocommit" mode should **not be considered for general use**. + If used, it should always be combined with the usage of + :meth:`.Session.begin` and :meth:`.Session.commit`, to ensure + a transaction demarcation. + + Executing queries outside of a demarcated transaction is a legacy mode + of usage, and can in some cases lead to concurrent connection + checkouts. + + In the absence of a demarcated transaction, the :class:`.Session` + cannot make appropriate decisions as to when autoflush should + occur nor when auto-expiration should occur, so these features + should be disabled with ``autoflush=False, expire_on_commit=False``. + +Modern usage of "autocommit" is for framework integrations that need to control +specifically when the "begin" state occurs. A session which is configured with +``autocommit=True`` may be placed into the "begin" state using the +:meth:`.Session.begin` method. +After the cycle completes upon :meth:`.Session.commit` or :meth:`.Session.rollback`, +connection and transaction resources are :term:`released` and the :class:`.Session` +goes back into "autocommit" mode, until :meth:`.Session.begin` is called again:: + + Session = sessionmaker(bind=engine, autocommit=True) + session = Session() + session.begin() + try: + item1 = session.query(Item).get(1) + item2 = session.query(Item).get(2) + item1.foo = 'bar' + item2.bar = 'foo' + session.commit() + except: + session.rollback() + raise + +The :meth:`.Session.begin` method also returns a transactional token which is +compatible with the Python 2.6 ``with`` statement:: + + Session = sessionmaker(bind=engine, autocommit=True) + session = Session() + with session.begin(): + item1 = session.query(Item).get(1) + item2 = session.query(Item).get(2) + item1.foo = 'bar' + item2.bar = 'foo' + +.. _session_subtransactions: + +Using Subtransactions with Autocommit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A subtransaction indicates usage of the :meth:`.Session.begin` method in conjunction with +the ``subtransactions=True`` flag. This produces a non-transactional, delimiting construct that +allows nesting of calls to :meth:`~.Session.begin` and :meth:`~.Session.commit`. +Its purpose is to allow the construction of code that can function within a transaction +both independently of any external code that starts a transaction, +as well as within a block that has already demarcated a transaction. + +``subtransactions=True`` is generally only useful in conjunction with +autocommit, and is equivalent to the pattern described at :ref:`connections_nested_transactions`, +where any number of functions can call :meth:`.Connection.begin` and :meth:`.Transaction.commit` +as though they are the initiator of the transaction, but in fact may be participating +in an already ongoing transaction:: + + # method_a starts a transaction and calls method_b + def method_a(session): + session.begin(subtransactions=True) + try: + method_b(session) + session.commit() # transaction is committed here + except: + session.rollback() # rolls back the transaction + raise + + # method_b also starts a transaction, but when + # called from method_a participates in the ongoing + # transaction. + def method_b(session): + session.begin(subtransactions=True) + try: + session.add(SomeObject('bat', 'lala')) + session.commit() # transaction is not committed yet + except: + session.rollback() # rolls back the transaction, in this case + # the one that was initiated in method_a(). + raise + + # create a Session and call method_a + session = Session(autocommit=True) + method_a(session) + session.close() + +Subtransactions are used by the :meth:`.Session.flush` process to ensure that the +flush operation takes place within a transaction, regardless of autocommit. When +autocommit is disabled, it is still useful in that it forces the :class:`.Session` +into a "pending rollback" state, as a failed flush cannot be resumed in mid-operation, +where the end user still maintains the "scope" of the transaction overall. + +.. _session_twophase: + +Enabling Two-Phase Commit +------------------------- + +For backends which support two-phase operaration (currently MySQL and +PostgreSQL), the session can be instructed to use two-phase commit semantics. +This will coordinate the committing of transactions across databases so that +the transaction is either committed or rolled back in all databases. You can +also :meth:`~.Session.prepare` the session for +interacting with transactions not managed by SQLAlchemy. To use two phase +transactions set the flag ``twophase=True`` on the session:: + + engine1 = create_engine('postgresql://db1') + engine2 = create_engine('postgresql://db2') + + Session = sessionmaker(twophase=True) + + # bind User operations to engine 1, Account operations to engine 2 + Session.configure(binds={User:engine1, Account:engine2}) + + session = Session() + + # .... work with accounts and users + + # commit. session will issue a flush to all DBs, and a prepare step to all DBs, + # before committing both transactions + session.commit() + +.. _session_external_transaction: + +Joining a Session into an External Transaction (such as for test suites) +======================================================================== + +If a :class:`.Connection` is being used which is already in a transactional +state (i.e. has a :class:`.Transaction` established), a :class:`.Session` can +be made to participate within that transaction by just binding the +:class:`.Session` to that :class:`.Connection`. The usual rationale for this +is a test suite that allows ORM code to work freely with a :class:`.Session`, +including the ability to call :meth:`.Session.commit`, where afterwards the +entire database interaction is rolled back:: + + from sqlalchemy.orm import sessionmaker + from sqlalchemy import create_engine + from unittest import TestCase + + # global application scope. create Session class, engine + Session = sessionmaker() + + engine = create_engine('postgresql://...') + + class SomeTest(TestCase): + def setUp(self): + # connect to the database + self.connection = engine.connect() + + # begin a non-ORM transaction + self.trans = self.connection.begin() + + # bind an individual Session to the connection + self.session = Session(bind=self.connection) + + def test_something(self): + # use the session in tests. + + self.session.add(Foo()) + self.session.commit() + + def tearDown(self): + self.session.close() + + # rollback - everything that happened with the + # Session above (including calls to commit()) + # is rolled back. + self.trans.rollback() + + # return connection to the Engine + self.connection.close() + +Above, we issue :meth:`.Session.commit` as well as +:meth:`.Transaction.rollback`. This is an example of where we take advantage +of the :class:`.Connection` object's ability to maintain *subtransactions*, or +nested begin/commit-or-rollback pairs where only the outermost begin/commit +pair actually commits the transaction, or if the outermost block rolls back, +everything is rolled back. + +.. topic:: Supporting Tests with Rollbacks + + The above recipe works well for any kind of database enabled test, except + for a test that needs to actually invoke :meth:`.Session.rollback` within + the scope of the test itself. The above recipe can be expanded, such + that the :class:`.Session` always runs all operations within the scope + of a SAVEPOINT, which is established at the start of each transaction, + so that tests can also rollback the "transaction" as well while still + remaining in the scope of a larger "transaction" that's never committed, + using two extra events:: + + from sqlalchemy import event + + class SomeTest(TestCase): + def setUp(self): + # connect to the database + self.connection = engine.connect() + + # begin a non-ORM transaction + self.trans = connection.begin() + + # bind an individual Session to the connection + self.session = Session(bind=self.connection) + + # start the session in a SAVEPOINT... + self.session.begin_nested() + + # then each time that SAVEPOINT ends, reopen it + @event.listens_for(self.session, "after_transaction_end") + def restart_savepoint(session, transaction): + if transaction.nested and not transaction._parent.nested: + session.begin_nested() + + + # ... the tearDown() method stays the same diff --git a/doc/build/orm/versioning.rst b/doc/build/orm/versioning.rst new file mode 100644 index 000000000..35304086d --- /dev/null +++ b/doc/build/orm/versioning.rst @@ -0,0 +1,253 @@ +.. _mapper_version_counter: + +Configuring a Version Counter +============================= + +The :class:`.Mapper` supports management of a :term:`version id column`, which +is a single table column that increments or otherwise updates its value +each time an ``UPDATE`` to the mapped table occurs. This value is checked each +time the ORM emits an ``UPDATE`` or ``DELETE`` against the row to ensure that +the value held in memory matches the database value. + +.. warning:: + + Because the versioning feature relies upon comparison of the **in memory** + record of an object, the feature only applies to the :meth:`.Session.flush` + process, where the ORM flushes individual in-memory rows to the database. + It does **not** take effect when performing + a multirow UPDATE or DELETE using :meth:`.Query.update` or :meth:`.Query.delete` + methods, as these methods only emit an UPDATE or DELETE statement but otherwise + do not have direct access to the contents of those rows being affected. + +The purpose of this feature is to detect when two concurrent transactions +are modifying the same row at roughly the same time, or alternatively to provide +a guard against the usage of a "stale" row in a system that might be re-using +data from a previous transaction without refreshing (e.g. if one sets ``expire_on_commit=False`` +with a :class:`.Session`, it is possible to re-use the data from a previous +transaction). + +.. topic:: Concurrent transaction updates + + When detecting concurrent updates within transactions, it is typically the + case that the database's transaction isolation level is below the level of + :term:`repeatable read`; otherwise, the transaction will not be exposed + to a new row value created by a concurrent update which conflicts with + the locally updated value. In this case, the SQLAlchemy versioning + feature will typically not be useful for in-transaction conflict detection, + though it still can be used for cross-transaction staleness detection. + + The database that enforces repeatable reads will typically either have locked the + target row against a concurrent update, or is employing some form + of multi version concurrency control such that it will emit an error + when the transaction is committed. SQLAlchemy's version_id_col is an alternative + which allows version tracking to occur for specific tables within a transaction + that otherwise might not have this isolation level set. + + .. seealso:: + + `Repeatable Read Isolation Level `_ - Postgresql's implementation of repeatable read, including a description of the error condition. + +Simple Version Counting +----------------------- + +The most straightforward way to track versions is to add an integer column +to the mapped table, then establish it as the ``version_id_col`` within the +mapper options:: + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + version_id = Column(Integer, nullable=False) + name = Column(String(50), nullable=False) + + __mapper_args__ = { + "version_id_col": version_id + } + +Above, the ``User`` mapping tracks integer versions using the column +``version_id``. When an object of type ``User`` is first flushed, the +``version_id`` column will be given a value of "1". Then, an UPDATE +of the table later on will always be emitted in a manner similar to the +following:: + + UPDATE user SET version_id=:version_id, name=:name + WHERE user.id = :user_id AND user.version_id = :user_version_id + {"name": "new name", "version_id": 2, "user_id": 1, "user_version_id": 1} + +The above UPDATE statement is updating the row that not only matches +``user.id = 1``, it also is requiring that ``user.version_id = 1``, where "1" +is the last version identifier we've been known to use on this object. +If a transaction elsewhere has modified the row independently, this version id +will no longer match, and the UPDATE statement will report that no rows matched; +this is the condition that SQLAlchemy tests, that exactly one row matched our +UPDATE (or DELETE) statement. If zero rows match, that indicates our version +of the data is stale, and a :exc:`.StaleDataError` is raised. + +.. _custom_version_counter: + +Custom Version Counters / Types +------------------------------- + +Other kinds of values or counters can be used for versioning. Common types include +dates and GUIDs. When using an alternate type or counter scheme, SQLAlchemy +provides a hook for this scheme using the ``version_id_generator`` argument, +which accepts a version generation callable. This callable is passed the value of the current +known version, and is expected to return the subsequent version. + +For example, if we wanted to track the versioning of our ``User`` class +using a randomly generated GUID, we could do this (note that some backends +support a native GUID type, but we illustrate here using a simple string):: + + import uuid + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + version_uuid = Column(String(32)) + name = Column(String(50), nullable=False) + + __mapper_args__ = { + 'version_id_col':version_uuid, + 'version_id_generator':lambda version: uuid.uuid4().hex + } + +The persistence engine will call upon ``uuid.uuid4()`` each time a +``User`` object is subject to an INSERT or an UPDATE. In this case, our +version generation function can disregard the incoming value of ``version``, +as the ``uuid4()`` function +generates identifiers without any prerequisite value. If we were using +a sequential versioning scheme such as numeric or a special character system, +we could make use of the given ``version`` in order to help determine the +subsequent value. + +.. seealso:: + + :ref:`custom_guid_type` + +.. _server_side_version_counter: + +Server Side Version Counters +---------------------------- + +The ``version_id_generator`` can also be configured to rely upon a value +that is generated by the database. In this case, the database would need +some means of generating new identifiers when a row is subject to an INSERT +as well as with an UPDATE. For the UPDATE case, typically an update trigger +is needed, unless the database in question supports some other native +version identifier. The Postgresql database in particular supports a system +column called `xmin `_ +which provides UPDATE versioning. We can make use +of the Postgresql ``xmin`` column to version our ``User`` +class as follows:: + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + name = Column(String(50), nullable=False) + xmin = Column("xmin", Integer, system=True) + + __mapper_args__ = { + 'version_id_col': xmin, + 'version_id_generator': False + } + +With the above mapping, the ORM will rely upon the ``xmin`` column for +automatically providing the new value of the version id counter. + +.. topic:: creating tables that refer to system columns + + In the above scenario, as ``xmin`` is a system column provided by Postgresql, + we use the ``system=True`` argument to mark it as a system-provided + column, omitted from the ``CREATE TABLE`` statement. + + +The ORM typically does not actively fetch the values of database-generated +values when it emits an INSERT or UPDATE, instead leaving these columns as +"expired" and to be fetched when they are next accessed, unless the ``eager_defaults`` +:func:`.mapper` flag is set. However, when a +server side version column is used, the ORM needs to actively fetch the newly +generated value. This is so that the version counter is set up *before* +any concurrent transaction may update it again. This fetching is also +best done simultaneously within the INSERT or UPDATE statement using :term:`RETURNING`, +otherwise if emitting a SELECT statement afterwards, there is still a potential +race condition where the version counter may change before it can be fetched. + +When the target database supports RETURNING, an INSERT statement for our ``User`` class will look +like this:: + + INSERT INTO "user" (name) VALUES (%(name)s) RETURNING "user".id, "user".xmin + {'name': 'ed'} + +Where above, the ORM can acquire any newly generated primary key values along +with server-generated version identifiers in one statement. When the backend +does not support RETURNING, an additional SELECT must be emitted for **every** +INSERT and UPDATE, which is much less efficient, and also introduces the possibility of +missed version counters:: + + INSERT INTO "user" (name) VALUES (%(name)s) + {'name': 'ed'} + + SELECT "user".version_id AS user_version_id FROM "user" where + "user".id = :param_1 + {"param_1": 1} + +It is *strongly recommended* that server side version counters only be used +when absolutely necessary and only on backends that support :term:`RETURNING`, +e.g. Postgresql, Oracle, SQL Server (though SQL Server has +`major caveats `_ when triggers are used), Firebird. + +.. versionadded:: 0.9.0 + + Support for server side version identifier tracking. + +Programmatic or Conditional Version Counters +--------------------------------------------- + +When ``version_id_generator`` is set to False, we can also programmatically +(and conditionally) set the version identifier on our object in the same way +we assign any other mapped attribute. Such as if we used our UUID example, but +set ``version_id_generator`` to ``False``, we can set the version identifier +at our choosing:: + + import uuid + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + version_uuid = Column(String(32)) + name = Column(String(50), nullable=False) + + __mapper_args__ = { + 'version_id_col':version_uuid, + 'version_id_generator': False + } + + u1 = User(name='u1', version_uuid=uuid.uuid4()) + + session.add(u1) + + session.commit() + + u1.name = 'u2' + u1.version_uuid = uuid.uuid4() + + session.commit() + +We can update our ``User`` object without incrementing the version counter +as well; the value of the counter will remain unchanged, and the UPDATE +statement will still check against the previous value. This may be useful +for schemes where only certain classes of UPDATE are sensitive to concurrency +issues:: + + # will leave version_uuid unchanged + u1.name = 'u3' + session.commit() + +.. versionadded:: 0.9.0 + + Support for programmatic and conditional version identifier tracking. + diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index 34f031b0b..3c26bea70 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ -mako changelog>=0.3.4 sphinx-paramlinks>=0.2.2 +zzzeeksphinx>=1.0.1 diff --git a/doc/build/static/detectmobile.js b/doc/build/static/detectmobile.js deleted file mode 100644 index f86b2d650..000000000 --- a/doc/build/static/detectmobile.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * jQuery.browser.mobile (http://detectmobilebrowser.com/) - * - * jQuery.browser.mobile will be true if the browser is a mobile device - * - **/ -(function(a){(jQuery.browser=jQuery.browser||{}).mobile=/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino/i.test(a)||/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(a.substr(0,4))})(navigator.userAgent||navigator.vendor||window.opera); \ No newline at end of file diff --git a/doc/build/static/docs.css b/doc/build/static/docs.css deleted file mode 100644 index e854d34c2..000000000 --- a/doc/build/static/docs.css +++ /dev/null @@ -1,673 +0,0 @@ -/* global */ - -.body-background { - background-color: #FDFBFC; -} - -body { - background-color: #FDFBFC; - margin:0 38px; - color:#333333; -} - -a { - font-weight:normal; - text-decoration:none; -} - -form { - display:inline; -} - -/* hyperlinks */ - -a:link, a:visited, a:active { - /*color:#0000FF;*/ - color: #990000; -} -a:hover { - color: #FF0000; - /*color:#700000;*/ - text-decoration:underline; -} - -/* paragraph links after sections. - These aren't visible until hovering - over the tag, then have a - "reverse video" effect over the actual - link - */ - -a.headerlink { - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; - visibility: hidden; -} - -h1:hover > a.headerlink, -h2:hover > a.headerlink, -h3:hover > a.headerlink, -h4:hover > a.headerlink, -h5:hover > a.headerlink, -h6:hover > a.headerlink, -dt:hover > a.headerlink { - visibility: visible; -} - -a.headerlink:hover { - background-color: #990000; - color: white; -} - - -/* Container setup */ - -#docs-container { - max-width:1000px; - margin: 0 auto; - position: relative; -} - - -/* header/footer elements */ - -#docs-header h1 { - font-size:20px; - color: #222222; - margin: 0; - padding: 0; -} - -#docs-header { - font-family:Verdana,sans-serif; - - font-size:.9em; - position: relative; -} - -#docs-sidebar-popout, -#docs-bottom-navigation, -#index-nav { - font-family: Verdana, sans-serif; - background-color: #FBFBEE; - border: solid 1px #CCC; - font-size:.8em; -} - -#docs-bottom-navigation, -#index-nav { - padding:10px; -} - -#docs-sidebar-popout { - font-size:.75em; -} - -#docs-sidebar-popout p, -#docs-sidebar-popout form { - margin:5px 0 5px 0px; -} - -#docs-sidebar-popout h3 { - margin:0 0 10px 0; -} - - -#docs-version-header { - position: absolute; - right: 0; - bottom: 0; -} - -.docs-navigation-links { - font-family:Verdana,sans-serif; -} - -#docs-bottom-navigation { - float:right; - margin: 1em 0 1em 5px; -} - -#docs-copyright { - font-size:.85em; - padding:5px 0px; -} - -#docs-header h1, -#docs-top-navigation h1, -#docs-top-navigation h2 { - font-family:Tahoma,Geneva,sans-serif; - font-weight:normal; -} - -#docs-top-navigation h2 { - margin:16px 4px 7px 5px; - font-size:1.6em; -} - -#docs-top-page-control { - position: absolute; - right: 20px; - bottom: 14px; -} - -#docs-top-page-control ul { - padding:0; - margin:0; -} - -#docs-top-page-control li { - font-size:.9em; - list-style-type:none; - padding:1px 8px; -} - - -#docs-container .version-num { - font-weight: bold; -} - - -/* content container, sidebar */ - -#docs-body-container { -} - -#docs-body, -#docs-sidebar, -#index-nav - { - /*font-family: helvetica, arial, sans-serif; - font-size:.9em;*/ - - font-family: Verdana, sans-serif; - font-size:.85em; - line-height:1.5em; - -} - -#docs-body { - min-height: 700px; -} - -#docs-sidebar > ul { - font-size:.85em; -} - -#fixed-sidebar { - position: relative; -} - -#fixed-sidebar.withsidebar { - float: left; - width:224px; -} - -#fixed-sidebar.preautomated { - position: fixed; - float: none; - top:0; - bottom: 0; -} - -#fixed-sidebar.automated { - position: fixed; - float: none; - top: 120px; - min-height: 0; -} - - -#docs-sidebar { - font-size:.85em; - - border: solid 1px #CCC; - - z-index: 3; - background-color: #EFEFEF; -} - -#index-nav { - position: relative; - margin-top:10px; - padding:0 10px; -} - -#index-nav form { - padding-top:10px; - float:right; -} - -#sidebar-paginate { - position: absolute; - bottom: 4.5em; - left: 10px; -} - -#sidebar-topnav { - position: absolute; - bottom: 3em; - left: 10px; -} - -#sidebar-search { - position: absolute; - bottom: 1em; - left: 10px; -} - -#docs-sidebar { - top: 132px; - bottom: 0; - min-height: 0; - overflow-y: auto; - margin-top:5px; - width:212px; - padding-left:10px; -} - -#docs-sidebar-popout { - height:120px; - max-height: 120px; - width:212px; - padding-left:10px; - padding-top:10px; - position: relative; -} - - -#fixed-sidebar.preautomated #docs-sidebar, -#fixed-sidebar.preautomated #docs-sidebar-popout { - position:absolute; -} - -#fixed-sidebar.preautomated #docs-sidebar:after { - content: " "; - display:block; - height: 150px; -} - - -#docs-sidebar.preautomated { - position: fixed; -} - -#docs-sidebar.automated { - position: fixed; - float: none; - top: 120px; - min-height: 0; -} - - -#docs-sidebar h3, #docs-sidebar h4 { - background-color: #DDDDDD; - color: #222222; - font-family: Verdana,sans-serif; - font-size: 1.1em; - font-weight: normal; - margin: 10px 0 0 -15px; - padding: 5px 10px 5px 15px; - text-shadow: 1px 1px 0 white; - /*width:210px;*/ -} - -#docs-sidebar h3:first-child { - margin-top: 0px; -} - -#docs-sidebar h3 a, #docs-sidebar h4 a { - color: #222222; -} -#docs-sidebar ul { - margin: 10px 10px 10px 0px; - padding: 0; - list-style: none outside none; -} - - -#docs-sidebar ul ul { - margin-bottom: 0; - margin-top: 0; - list-style: square outside none; - margin-left: 20px; -} - - - - -#docs-body { - background-color:#FFFFFF; - padding:1px 10px 10px 10px; - - border: solid 1px #CCC; - margin-top:10px; -} - -#docs-body.withsidebar { - margin-left: 230px; -} - - -#docs-body h1, -#docs-body h2, -#docs-body h3, -#docs-body h4 { - font-family:Helvetica, Arial, sans-serif; -} - -#docs-body #sqlalchemy-documentation h1 { - /* hide the

for each content section. */ - display:none; - font-size:2.0em; -} - - -#docs-body h2 { - font-size:1.8em; - border-top:1px solid; - /*border-bottom:1px solid;*/ - padding-top:20px; -} - -#sqlalchemy-documentation h2 { - border-top:none; - padding-top:0; -} -#docs-body h3 { - font-size:1.4em; -} - -/* SQL popup, code styles */ - -.highlight { - background:none; -} - -#docs-container pre { - font-size:1.2em; -} - -#docs-container .pre { - font-size:1.1em; -} - -#docs-container pre { - background-color: #f0f0f0; - border: solid 1px #ccc; - box-shadow: 2px 2px 3px #DFDFDF; - padding:10px; - margin: 5px 0px 5px 0px; - overflow:auto; - line-height:1.3em; -} - -.popup_sql, .show_sql -{ - background-color: #FBFBEE; - padding:5px 10px; - margin:10px -5px; - border:1px dashed; -} - -/* the [SQL] links used to display SQL */ -#docs-container .sql_link -{ - font-weight:normal; - font-family: arial, sans-serif; - font-size:.9em; - text-transform: uppercase; - color:#990000; - border:1px solid; - padding:1px 2px 1px 2px; - margin:0px 10px 0px 15px; - float:right; - line-height:1.2em; -} - -#docs-container a.sql_link, -#docs-container .sql_link -{ - text-decoration: none; - padding:1px 2px; -} - -#docs-container a.sql_link:hover { - text-decoration: none; - color:#fff; - border:1px solid #900; - background-color: #900; -} - -/* changeset stuff */ - -#docs-container a.changeset-link { - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; -} - -/* docutils-specific elements */ - -th.field-name { - text-align:right; -} - -div.section { -} - -div.note, div.warning, p.deprecated, div.topic, div.admonition { - background-color:#EEFFEF; -} - -.footnote { - font-size: .95em; -} - -div.faq { - background-color: #EFEFEF; -} - -div.faq ul { - list-style: square outside none; -} - -div.admonition, div.topic, .deprecated, .versionadded, .versionchanged { - border:1px solid #CCCCCC; - padding:5px 10px; - font-size:.9em; - margin-top:5px; - box-shadow: 2px 2px 3px #DFDFDF; -} - -div.sidebar { - background-color: #FFFFEE; - border: 1px solid #DDDDBB; - float: right; - margin: 10px 0 10px 1em; - padding: 7px 7px 0; - width: 40%; - font-size:.9em; -} - -p.sidebar-title { - font-weight: bold; -} - -/* grrr sphinx changing your document structures, removing classes.... */ - -.versionadded .versionmodified, -.versionchanged .versionmodified, -.deprecated .versionmodified, -.versionadded > p:first-child > span:first-child, -.versionchanged > p:first-child > span:first-child, -.deprecated > p:first-child > span:first-child -{ - background-color: #ECF0F3; - color: #990000; - font-style: italic; -} - - -div.inherited-member { - border:1px solid #CCCCCC; - padding:5px 5px; - font-size:.9em; - box-shadow: 2px 2px 3px #DFDFDF; -} - -div.warning .admonition-title { - color:#FF0000; -} - -div.admonition .admonition-title, div.topic .topic-title { - font-weight:bold; -} - -.viewcode-back, .viewcode-link { - float:right; -} - -dl.function > dt, -dl.attribute > dt, -dl.classmethod > dt, -dl.method > dt, -dl.class > dt, -dl.exception > dt -{ - background-color: #EFEFEF; - margin:25px -10px 10px 10px; - padding: 0px 10px; -} - - -dl.glossary > dt { - font-weight:bold; - font-size:1.1em; - padding-top:10px; -} - - -dt:target, span.highlight { - background-color:#FBE54E; -} - -a.headerlink { - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; - visibility: hidden; -} - -h1:hover > a.headerlink, -h2:hover > a.headerlink, -h3:hover > a.headerlink, -h4:hover > a.headerlink, -h5:hover > a.headerlink, -h6:hover > a.headerlink, -dt:hover > a.headerlink { - visibility: visible; -} - -a.headerlink:hover { - background-color: #00f; - color: white; -} - -.clearboth { - clear:both; -} - -tt.descname { - background-color:transparent; - font-size:1.2em; - font-weight:bold; -} - -tt.descclassname { - background-color:transparent; -} - -tt { - background-color:#ECF0F3; - padding:0 1px; -} - -/* syntax highlighting overrides */ -.k, .kn {color:#0908CE;} -.o {color:#BF0005;} -.go {color:#804049;} - - -/* special "index page" sections - with specific formatting -*/ - -div#sqlalchemy-documentation { - font-size:.95em; -} -div#sqlalchemy-documentation em { - font-style:normal; -} -div#sqlalchemy-documentation .rubric{ - font-size:14px; - background-color:#EEFFEF; - padding:5px; - border:1px solid #BFBFBF; -} -div#sqlalchemy-documentation a, div#sqlalchemy-documentation li { - padding:5px 0px; -} - -div#getting-started { - border-bottom:1px solid; -} - -div#sqlalchemy-documentation div#sqlalchemy-orm { - float:left; - width:48%; -} - -div#sqlalchemy-documentation div#sqlalchemy-core { - float:left; - width:48%; - margin:0; - padding-left:10px; - border-left:1px solid; -} - -div#dialect-documentation { - border-top:1px solid; - /*clear:left;*/ -} - -div .versionwarning, -div .version-warning { - font-size:12px; - font-color:red; - border:1px solid; - padding:4px 4px; - margin:8px 0px 2px 0px; - background:#FFBBBB; -} - -/*div .event-signatures { - background-color:#F0F0FD; - padding:0 10px; - border:1px solid #BFBFBF; -}*/ - -/*dl div.floatything { - display:none; - position:fixed; - top:25px; - left:40px; - font-size:.95em; - font-weight: bold; - border:1px solid; - background-color: #FFF; -} -dl:hover div.floatything { - display:block; -}*/ diff --git a/doc/build/static/init.js b/doc/build/static/init.js deleted file mode 100644 index 4bcb4411d..000000000 --- a/doc/build/static/init.js +++ /dev/null @@ -1,44 +0,0 @@ - -function initSQLPopups() { - $('div.popup_sql').hide(); - $('a.sql_link').click(function() { - $(this).nextAll('div.popup_sql:first').toggle(); - return false; - }); -} - -var automatedBreakpoint = -1; - -function initFloatyThings() { - - automatedBreakpoint = $("#docs-container").position().top + $("#docs-top-navigation-container").height(); - - $("#fixed-sidebar.withsidebar").addClass("preautomated"); - - - function setScroll() { - - var scrolltop = $(window).scrollTop(); - if (scrolltop >= automatedBreakpoint) { - $("#fixed-sidebar.withsidebar").css("top", 5); - } - else { - $("#fixed-sidebar.withsidebar").css( - "top", $("#docs-body").offset().top - Math.max(scrolltop, 0)); - } - - - } - $(window).scroll(setScroll) - - setScroll(); -} - - -$(document).ready(function() { - initSQLPopups(); - if (!$.browser.mobile) { - initFloatyThings(); - } -}); - diff --git a/doc/build/templates/genindex.mako b/doc/build/templates/genindex.mako deleted file mode 100644 index 9ea6795bc..000000000 --- a/doc/build/templates/genindex.mako +++ /dev/null @@ -1,77 +0,0 @@ -<%inherit file="layout.mako"/> - -<%block name="show_title" filter="util.striptags"> - ${_('Index')} - - -

${_('Index')}

- - % for i, (key, dummy) in enumerate(genindexentries): - ${i != 0 and '| ' or ''}${key} - % endfor - -
- - % for i, (key, entries) in enumerate(genindexentries): -

${key}

-
-
- <% - breakat = genindexcounts[i] // 2 - numcols = 1 - numitems = 0 - %> -% for entryname, (links, subitems) in entries: - -
- % if links: - ${entryname|h} - % for unknown, link in links[1:]: - , [${i}] - % endfor - % else: - ${entryname|h} - % endif -
- - % if subitems: -
- % for subentryname, subentrylinks in subitems: -
${subentryname|h} - % for j, (unknown, link) in enumerate(subentrylinks[1:]): - [${j}] - % endfor -
- % endfor -
- % endif - - <% - numitems = numitems + 1 + len(subitems) - %> - % if numcols <2 and numitems > breakat: - <% - numcols = numcols + 1 - %> -
- % endif - -% endfor -
-
-% endfor - -<%def name="sidebarrel()"> -% if split_index: -

${_('Index')}

-

- % for i, (key, dummy) in enumerate(genindexentries): - ${i > 0 and '| ' or ''} - ${key} - % endfor -

- -

${_('Full index on one page')}

-% endif - ${parent.sidebarrel()} - diff --git a/doc/build/templates/layout.mako b/doc/build/templates/layout.mako deleted file mode 100644 index 23e57129b..000000000 --- a/doc/build/templates/layout.mako +++ /dev/null @@ -1,243 +0,0 @@ -## coding: utf-8 - -<%! - local_script_files = [] - - default_css_files = [ - '_static/pygments.css', - '_static/docs.css', - ] -%> - - -<%doc> - Structural elements are all prefixed with "docs-" - to prevent conflicts when the structure is integrated into the - main site. - - docs-container -> - docs-top-navigation-container -> - docs-header -> - docs-version-header - docs-top-navigation - docs-top-page-control - docs-navigation-banner - docs-body-container -> - docs-sidebar - docs-body - docs-bottom-navigation - docs-copyright - - -<%inherit file="${context['base']}"/> - -<% - if builder == 'epub': - next.body() - return -%> - - -<% -withsidebar = bool(toc) and current_page_name != 'index' -%> - -<%block name="head_title"> - % if current_page_name != 'index': - ${capture(self.show_title) | util.striptags} — - % endif - ${docstitle|h} - - - -
- - -<%block name="headers"> - - ${parent.headers()} - - - - - - - % for scriptfile in script_files + self.attr.local_script_files: - - % endfor - - - - - % if hasdoc('about'): - - % endif - - - % if hasdoc('copyright'): - - % endif - - % if parents: - - % endif - % if nexttopic: - - % endif - % if prevtopic: - - % endif - - - - - -
-
-
- Release: ${release} | Release Date: ${release_date} -
- -

${docstitle|h}

- -
-
- -
- -
- - % if not withsidebar: -
-
- - - -
- -

- Contents | - Index - % if pdf_url: - | Download as PDF - % endif -

- -
- % endif - - % if withsidebar: -
-

${docstitle|h}

- - - - - - - -
- -
- -

\ - <%block name="show_title"> - ${title} - -

- ${toc} - - % if rtd: -

Project Versions

-
    -
- % endif - - -
- % endif - -
- - <%doc> -
- ${docstitle|h} - % if parents: - % for parent in parents: - » ${parent['title']} - % endfor - % endif - % if current_page_name != 'index': - » ${self.show_title()} - % endif - -

- <%block name="show_title"> - ${title} - -

- -
- - -
- ${next.body()} -
- -
- - - -
diff --git a/doc/build/templates/page.mako b/doc/build/templates/page.mako deleted file mode 100644 index e0f98cf64..000000000 --- a/doc/build/templates/page.mako +++ /dev/null @@ -1,2 +0,0 @@ -<%inherit file="layout.mako"/> -${body| util.strip_toplevel_anchors} \ No newline at end of file diff --git a/doc/build/templates/search.mako b/doc/build/templates/search.mako deleted file mode 100644 index d0aa3d825..000000000 --- a/doc/build/templates/search.mako +++ /dev/null @@ -1,21 +0,0 @@ -<%inherit file="layout.mako"/> - -<%! - local_script_files = ['_static/searchtools.js'] -%> -<%block name="show_title"> - ${_('Search')} - - -<%block name="headers"> - ${parent.headers()} - - - -
- -<%block name="footer"> - ${parent.footer()} - diff --git a/doc/build/templates/static_base.mako b/doc/build/templates/static_base.mako deleted file mode 100644 index 9eb5ec046..000000000 --- a/doc/build/templates/static_base.mako +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - ${metatags and metatags or ''} - - <%block name="head_title"> - </%block> - - - <%block name="css"> - - % for cssfile in self.attr.default_css_files + css_files: - - % endfor - - - - <%block name="headers"/> - - - ${next.body()} - <%block name="footer"/> - - - - diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index e79299527..e0b2875e8 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -108,7 +108,7 @@ file-based architecture and additionally will usually require workarounds to work when using the pysqlite driver. Transaction Isolation Level -=========================== +---------------------------- SQLite supports "transaction isolation" in a non-standard way, along two axes. One is that of the `PRAGMA read_uncommitted `_ @@ -141,7 +141,7 @@ by *not even emitting BEGIN* until the first write operation. for techniques to work around this behavior. SAVEPOINT Support -================= +---------------------------- SQLite supports SAVEPOINTs, which only function once a transaction is begun. SQLAlchemy's SAVEPOINT support is available using the @@ -157,7 +157,7 @@ won't work at all with pysqlite unless workarounds are taken. for techniques to work around this behavior. Transactional DDL -================= +---------------------------- The SQLite database supports transactional :term:`DDL` as well. In this case, the pysqlite driver is not only failing to start transactions, diff --git a/lib/sqlalchemy/ext/declarative/__init__.py b/lib/sqlalchemy/ext/declarative/__init__.py index 2b611252a..cbde6f9d2 100644 --- a/lib/sqlalchemy/ext/declarative/__init__.py +++ b/lib/sqlalchemy/ext/declarative/__init__.py @@ -5,1377 +5,6 @@ # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -""" -Synopsis -======== - -SQLAlchemy object-relational configuration involves the -combination of :class:`.Table`, :func:`.mapper`, and class -objects to define a mapped class. -:mod:`~sqlalchemy.ext.declarative` allows all three to be -expressed at once within the class declaration. As much as -possible, regular SQLAlchemy schema and ORM constructs are -used directly, so that configuration between "classical" ORM -usage and declarative remain highly similar. - -As a simple example:: - - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class SomeClass(Base): - __tablename__ = 'some_table' - id = Column(Integer, primary_key=True) - name = Column(String(50)) - -Above, the :func:`declarative_base` callable returns a new base class from -which all mapped classes should inherit. When the class definition is -completed, a new :class:`.Table` and :func:`.mapper` will have been generated. - -The resulting table and mapper are accessible via -``__table__`` and ``__mapper__`` attributes on the -``SomeClass`` class:: - - # access the mapped Table - SomeClass.__table__ - - # access the Mapper - SomeClass.__mapper__ - -Defining Attributes -=================== - -In the previous example, the :class:`.Column` objects are -automatically named with the name of the attribute to which they are -assigned. - -To name columns explicitly with a name distinct from their mapped attribute, -just give the column a name. Below, column "some_table_id" is mapped to the -"id" attribute of `SomeClass`, but in SQL will be represented as -"some_table_id":: - - class SomeClass(Base): - __tablename__ = 'some_table' - id = Column("some_table_id", Integer, primary_key=True) - -Attributes may be added to the class after its construction, and they will be -added to the underlying :class:`.Table` and -:func:`.mapper` definitions as appropriate:: - - SomeClass.data = Column('data', Unicode) - SomeClass.related = relationship(RelatedInfo) - -Classes which are constructed using declarative can interact freely -with classes that are mapped explicitly with :func:`.mapper`. - -It is recommended, though not required, that all tables -share the same underlying :class:`~sqlalchemy.schema.MetaData` object, -so that string-configured :class:`~sqlalchemy.schema.ForeignKey` -references can be resolved without issue. - -Accessing the MetaData -======================= - -The :func:`declarative_base` base class contains a -:class:`.MetaData` object where newly defined -:class:`.Table` objects are collected. This object is -intended to be accessed directly for -:class:`.MetaData`-specific operations. Such as, to issue -CREATE statements for all tables:: - - engine = create_engine('sqlite://') - Base.metadata.create_all(engine) - -:func:`declarative_base` can also receive a pre-existing -:class:`.MetaData` object, which allows a -declarative setup to be associated with an already -existing traditional collection of :class:`~sqlalchemy.schema.Table` -objects:: - - mymetadata = MetaData() - Base = declarative_base(metadata=mymetadata) - - -.. _declarative_configuring_relationships: - -Configuring Relationships -========================= - -Relationships to other classes are done in the usual way, with the added -feature that the class specified to :func:`~sqlalchemy.orm.relationship` -may be a string name. The "class registry" associated with ``Base`` -is used at mapper compilation time to resolve the name into the actual -class object, which is expected to have been defined once the mapper -configuration is used:: - - class User(Base): - __tablename__ = 'users' - - id = Column(Integer, primary_key=True) - name = Column(String(50)) - addresses = relationship("Address", backref="user") - - class Address(Base): - __tablename__ = 'addresses' - - id = Column(Integer, primary_key=True) - email = Column(String(50)) - user_id = Column(Integer, ForeignKey('users.id')) - -Column constructs, since they are just that, are immediately usable, -as below where we define a primary join condition on the ``Address`` -class using them:: - - class Address(Base): - __tablename__ = 'addresses' - - id = Column(Integer, primary_key=True) - email = Column(String(50)) - user_id = Column(Integer, ForeignKey('users.id')) - user = relationship(User, primaryjoin=user_id == User.id) - -In addition to the main argument for :func:`~sqlalchemy.orm.relationship`, -other arguments which depend upon the columns present on an as-yet -undefined class may also be specified as strings. These strings are -evaluated as Python expressions. The full namespace available within -this evaluation includes all classes mapped for this declarative base, -as well as the contents of the ``sqlalchemy`` package, including -expression functions like :func:`~sqlalchemy.sql.expression.desc` and -:attr:`~sqlalchemy.sql.expression.func`:: - - class User(Base): - # .... - addresses = relationship("Address", - order_by="desc(Address.email)", - primaryjoin="Address.user_id==User.id") - -For the case where more than one module contains a class of the same name, -string class names can also be specified as module-qualified paths -within any of these string expressions:: - - class User(Base): - # .... - addresses = relationship("myapp.model.address.Address", - order_by="desc(myapp.model.address.Address.email)", - primaryjoin="myapp.model.address.Address.user_id==" - "myapp.model.user.User.id") - -The qualified path can be any partial path that removes ambiguity between -the names. For example, to disambiguate between -``myapp.model.address.Address`` and ``myapp.model.lookup.Address``, -we can specify ``address.Address`` or ``lookup.Address``:: - - class User(Base): - # .... - addresses = relationship("address.Address", - order_by="desc(address.Address.email)", - primaryjoin="address.Address.user_id==" - "User.id") - -.. versionadded:: 0.8 - module-qualified paths can be used when specifying string arguments - with Declarative, in order to specify specific modules. - -Two alternatives also exist to using string-based attributes. A lambda -can also be used, which will be evaluated after all mappers have been -configured:: - - class User(Base): - # ... - addresses = relationship(lambda: Address, - order_by=lambda: desc(Address.email), - primaryjoin=lambda: Address.user_id==User.id) - -Or, the relationship can be added to the class explicitly after the classes -are available:: - - User.addresses = relationship(Address, - primaryjoin=Address.user_id==User.id) - - - -.. _declarative_many_to_many: - -Configuring Many-to-Many Relationships -====================================== - -Many-to-many relationships are also declared in the same way -with declarative as with traditional mappings. The -``secondary`` argument to -:func:`.relationship` is as usual passed a -:class:`.Table` object, which is typically declared in the -traditional way. The :class:`.Table` usually shares -the :class:`.MetaData` object used by the declarative base:: - - keywords = Table( - 'keywords', Base.metadata, - Column('author_id', Integer, ForeignKey('authors.id')), - Column('keyword_id', Integer, ForeignKey('keywords.id')) - ) - - class Author(Base): - __tablename__ = 'authors' - id = Column(Integer, primary_key=True) - keywords = relationship("Keyword", secondary=keywords) - -Like other :func:`~sqlalchemy.orm.relationship` arguments, a string is accepted -as well, passing the string name of the table as defined in the -``Base.metadata.tables`` collection:: - - class Author(Base): - __tablename__ = 'authors' - id = Column(Integer, primary_key=True) - keywords = relationship("Keyword", secondary="keywords") - -As with traditional mapping, its generally not a good idea to use -a :class:`.Table` as the "secondary" argument which is also mapped to -a class, unless the :func:`.relationship` is declared with ``viewonly=True``. -Otherwise, the unit-of-work system may attempt duplicate INSERT and -DELETE statements against the underlying table. - -.. _declarative_sql_expressions: - -Defining SQL Expressions -======================== - -See :ref:`mapper_sql_expressions` for examples on declaratively -mapping attributes to SQL expressions. - -.. _declarative_table_args: - -Table Configuration -=================== - -Table arguments other than the name, metadata, and mapped Column -arguments are specified using the ``__table_args__`` class attribute. -This attribute accommodates both positional as well as keyword -arguments that are normally sent to the -:class:`~sqlalchemy.schema.Table` constructor. -The attribute can be specified in one of two forms. One is as a -dictionary:: - - class MyClass(Base): - __tablename__ = 'sometable' - __table_args__ = {'mysql_engine':'InnoDB'} - -The other, a tuple, where each argument is positional -(usually constraints):: - - class MyClass(Base): - __tablename__ = 'sometable' - __table_args__ = ( - ForeignKeyConstraint(['id'], ['remote_table.id']), - UniqueConstraint('foo'), - ) - -Keyword arguments can be specified with the above form by -specifying the last argument as a dictionary:: - - class MyClass(Base): - __tablename__ = 'sometable' - __table_args__ = ( - ForeignKeyConstraint(['id'], ['remote_table.id']), - UniqueConstraint('foo'), - {'autoload':True} - ) - -Using a Hybrid Approach with __table__ -======================================= - -As an alternative to ``__tablename__``, a direct -:class:`~sqlalchemy.schema.Table` construct may be used. The -:class:`~sqlalchemy.schema.Column` objects, which in this case require -their names, will be added to the mapping just like a regular mapping -to a table:: - - class MyClass(Base): - __table__ = Table('my_table', Base.metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)) - ) - -``__table__`` provides a more focused point of control for establishing -table metadata, while still getting most of the benefits of using declarative. -An application that uses reflection might want to load table metadata elsewhere -and pass it to declarative classes:: - - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - Base.metadata.reflect(some_engine) - - class User(Base): - __table__ = metadata.tables['user'] - - class Address(Base): - __table__ = metadata.tables['address'] - -Some configuration schemes may find it more appropriate to use ``__table__``, -such as those which already take advantage of the data-driven nature of -:class:`.Table` to customize and/or automate schema definition. - -Note that when the ``__table__`` approach is used, the object is immediately -usable as a plain :class:`.Table` within the class declaration body itself, -as a Python class is only another syntactical block. Below this is illustrated -by using the ``id`` column in the ``primaryjoin`` condition of a -:func:`.relationship`:: - - class MyClass(Base): - __table__ = Table('my_table', Base.metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)) - ) - - widgets = relationship(Widget, - primaryjoin=Widget.myclass_id==__table__.c.id) - -Similarly, mapped attributes which refer to ``__table__`` can be placed inline, -as below where we assign the ``name`` column to the attribute ``_name``, -generating a synonym for ``name``:: - - from sqlalchemy.ext.declarative import synonym_for - - class MyClass(Base): - __table__ = Table('my_table', Base.metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)) - ) - - _name = __table__.c.name - - @synonym_for("_name") - def name(self): - return "Name: %s" % _name - -Using Reflection with Declarative -================================= - -It's easy to set up a :class:`.Table` that uses ``autoload=True`` -in conjunction with a mapped class:: - - class MyClass(Base): - __table__ = Table('mytable', Base.metadata, - autoload=True, autoload_with=some_engine) - -However, one improvement that can be made here is to not -require the :class:`.Engine` to be available when classes are -being first declared. To achieve this, use the -:class:`.DeferredReflection` mixin, which sets up mappings -only after a special ``prepare(engine)`` step is called:: - - from sqlalchemy.ext.declarative import declarative_base, DeferredReflection - - Base = declarative_base(cls=DeferredReflection) - - class Foo(Base): - __tablename__ = 'foo' - bars = relationship("Bar") - - class Bar(Base): - __tablename__ = 'bar' - - # illustrate overriding of "bar.foo_id" to have - # a foreign key constraint otherwise not - # reflected, such as when using MySQL - foo_id = Column(Integer, ForeignKey('foo.id')) - - Base.prepare(e) - -.. versionadded:: 0.8 - Added :class:`.DeferredReflection`. - -Mapper Configuration -==================== - -Declarative makes use of the :func:`~.orm.mapper` function internally -when it creates the mapping to the declared table. The options -for :func:`~.orm.mapper` are passed directly through via the -``__mapper_args__`` class attribute. As always, arguments which reference -locally mapped columns can reference them directly from within the -class declaration:: - - from datetime import datetime - - class Widget(Base): - __tablename__ = 'widgets' - - id = Column(Integer, primary_key=True) - timestamp = Column(DateTime, nullable=False) - - __mapper_args__ = { - 'version_id_col': timestamp, - 'version_id_generator': lambda v:datetime.now() - } - -.. _declarative_inheritance: - -Inheritance Configuration -========================= - -Declarative supports all three forms of inheritance as intuitively -as possible. The ``inherits`` mapper keyword argument is not needed -as declarative will determine this from the class itself. The various -"polymorphic" keyword arguments are specified using ``__mapper_args__``. - -Joined Table Inheritance -~~~~~~~~~~~~~~~~~~~~~~~~ - -Joined table inheritance is defined as a subclass that defines its own -table:: - - class Person(Base): - __tablename__ = 'people' - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __tablename__ = 'engineers' - __mapper_args__ = {'polymorphic_identity': 'engineer'} - id = Column(Integer, ForeignKey('people.id'), primary_key=True) - primary_language = Column(String(50)) - -Note that above, the ``Engineer.id`` attribute, since it shares the -same attribute name as the ``Person.id`` attribute, will in fact -represent the ``people.id`` and ``engineers.id`` columns together, -with the "Engineer.id" column taking precedence if queried directly. -To provide the ``Engineer`` class with an attribute that represents -only the ``engineers.id`` column, give it a different attribute name:: - - class Engineer(Person): - __tablename__ = 'engineers' - __mapper_args__ = {'polymorphic_identity': 'engineer'} - engineer_id = Column('id', Integer, ForeignKey('people.id'), - primary_key=True) - primary_language = Column(String(50)) - - -.. versionchanged:: 0.7 joined table inheritance favors the subclass - column over that of the superclass, such as querying above - for ``Engineer.id``. Prior to 0.7 this was the reverse. - -.. _declarative_single_table: - -Single Table Inheritance -~~~~~~~~~~~~~~~~~~~~~~~~ - -Single table inheritance is defined as a subclass that does not have -its own table; you just leave out the ``__table__`` and ``__tablename__`` -attributes:: - - class Person(Base): - __tablename__ = 'people' - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __mapper_args__ = {'polymorphic_identity': 'engineer'} - primary_language = Column(String(50)) - -When the above mappers are configured, the ``Person`` class is mapped -to the ``people`` table *before* the ``primary_language`` column is -defined, and this column will not be included in its own mapping. -When ``Engineer`` then defines the ``primary_language`` column, the -column is added to the ``people`` table so that it is included in the -mapping for ``Engineer`` and is also part of the table's full set of -columns. Columns which are not mapped to ``Person`` are also excluded -from any other single or joined inheriting classes using the -``exclude_properties`` mapper argument. Below, ``Manager`` will have -all the attributes of ``Person`` and ``Manager`` but *not* the -``primary_language`` attribute of ``Engineer``:: - - class Manager(Person): - __mapper_args__ = {'polymorphic_identity': 'manager'} - golf_swing = Column(String(50)) - -The attribute exclusion logic is provided by the -``exclude_properties`` mapper argument, and declarative's default -behavior can be disabled by passing an explicit ``exclude_properties`` -collection (empty or otherwise) to the ``__mapper_args__``. - -Resolving Column Conflicts -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note above that the ``primary_language`` and ``golf_swing`` columns -are "moved up" to be applied to ``Person.__table__``, as a result of their -declaration on a subclass that has no table of its own. A tricky case -comes up when two subclasses want to specify *the same* column, as below:: - - class Person(Base): - __tablename__ = 'people' - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __mapper_args__ = {'polymorphic_identity': 'engineer'} - start_date = Column(DateTime) - - class Manager(Person): - __mapper_args__ = {'polymorphic_identity': 'manager'} - start_date = Column(DateTime) - -Above, the ``start_date`` column declared on both ``Engineer`` and ``Manager`` -will result in an error:: - - sqlalchemy.exc.ArgumentError: Column 'start_date' on class - conflicts with existing - column 'people.start_date' - -In a situation like this, Declarative can't be sure -of the intent, especially if the ``start_date`` columns had, for example, -different types. A situation like this can be resolved by using -:class:`.declared_attr` to define the :class:`.Column` conditionally, taking -care to return the **existing column** via the parent ``__table__`` if it -already exists:: - - from sqlalchemy.ext.declarative import declared_attr - - class Person(Base): - __tablename__ = 'people' - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __mapper_args__ = {'polymorphic_identity': 'engineer'} - - @declared_attr - def start_date(cls): - "Start date column, if not present already." - return Person.__table__.c.get('start_date', Column(DateTime)) - - class Manager(Person): - __mapper_args__ = {'polymorphic_identity': 'manager'} - - @declared_attr - def start_date(cls): - "Start date column, if not present already." - return Person.__table__.c.get('start_date', Column(DateTime)) - -Above, when ``Manager`` is mapped, the ``start_date`` column is -already present on the ``Person`` class. Declarative lets us return -that :class:`.Column` as a result in this case, where it knows to skip -re-assigning the same column. If the mapping is mis-configured such -that the ``start_date`` column is accidentally re-assigned to a -different table (such as, if we changed ``Manager`` to be joined -inheritance without fixing ``start_date``), an error is raised which -indicates an existing :class:`.Column` is trying to be re-assigned to -a different owning :class:`.Table`. - -.. versionadded:: 0.8 :class:`.declared_attr` can be used on a non-mixin - class, and the returned :class:`.Column` or other mapped attribute - will be applied to the mapping as any other attribute. Previously, - the resulting attribute would be ignored, and also result in a warning - being emitted when a subclass was created. - -.. versionadded:: 0.8 :class:`.declared_attr`, when used either with a - mixin or non-mixin declarative class, can return an existing - :class:`.Column` already assigned to the parent :class:`.Table`, - to indicate that the re-assignment of the :class:`.Column` should be - skipped, however should still be mapped on the target class, - in order to resolve duplicate column conflicts. - -The same concept can be used with mixin classes (see -:ref:`declarative_mixins`):: - - class Person(Base): - __tablename__ = 'people' - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class HasStartDate(object): - @declared_attr - def start_date(cls): - return cls.__table__.c.get('start_date', Column(DateTime)) - - class Engineer(HasStartDate, Person): - __mapper_args__ = {'polymorphic_identity': 'engineer'} - - class Manager(HasStartDate, Person): - __mapper_args__ = {'polymorphic_identity': 'manager'} - -The above mixin checks the local ``__table__`` attribute for the column. -Because we're using single table inheritance, we're sure that in this case, -``cls.__table__`` refers to ``People.__table__``. If we were mixing joined- -and single-table inheritance, we might want our mixin to check more carefully -if ``cls.__table__`` is really the :class:`.Table` we're looking for. - -Concrete Table Inheritance -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Concrete is defined as a subclass which has its own table and sets the -``concrete`` keyword argument to ``True``:: - - class Person(Base): - __tablename__ = 'people' - id = Column(Integer, primary_key=True) - name = Column(String(50)) - - class Engineer(Person): - __tablename__ = 'engineers' - __mapper_args__ = {'concrete':True} - id = Column(Integer, primary_key=True) - primary_language = Column(String(50)) - name = Column(String(50)) - -Usage of an abstract base class is a little less straightforward as it -requires usage of :func:`~sqlalchemy.orm.util.polymorphic_union`, -which needs to be created with the :class:`.Table` objects -before the class is built:: - - engineers = Table('engineers', Base.metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)), - Column('primary_language', String(50)) - ) - managers = Table('managers', Base.metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)), - Column('golf_swing', String(50)) - ) - - punion = polymorphic_union({ - 'engineer':engineers, - 'manager':managers - }, 'type', 'punion') - - class Person(Base): - __table__ = punion - __mapper_args__ = {'polymorphic_on':punion.c.type} - - class Engineer(Person): - __table__ = engineers - __mapper_args__ = {'polymorphic_identity':'engineer', 'concrete':True} - - class Manager(Person): - __table__ = managers - __mapper_args__ = {'polymorphic_identity':'manager', 'concrete':True} - -.. _declarative_concrete_helpers: - -Using the Concrete Helpers -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Helper classes provides a simpler pattern for concrete inheritance. -With these objects, the ``__declare_first__`` helper is used to configure the -"polymorphic" loader for the mapper after all subclasses have been declared. - -.. versionadded:: 0.7.3 - -An abstract base can be declared using the -:class:`.AbstractConcreteBase` class:: - - from sqlalchemy.ext.declarative import AbstractConcreteBase - - class Employee(AbstractConcreteBase, Base): - pass - -To have a concrete ``employee`` table, use :class:`.ConcreteBase` instead:: - - from sqlalchemy.ext.declarative import ConcreteBase - - class Employee(ConcreteBase, Base): - __tablename__ = 'employee' - employee_id = Column(Integer, primary_key=True) - name = Column(String(50)) - __mapper_args__ = { - 'polymorphic_identity':'employee', - 'concrete':True} - - -Either ``Employee`` base can be used in the normal fashion:: - - class Manager(Employee): - __tablename__ = 'manager' - employee_id = Column(Integer, primary_key=True) - name = Column(String(50)) - manager_data = Column(String(40)) - __mapper_args__ = { - 'polymorphic_identity':'manager', - 'concrete':True} - - class Engineer(Employee): - __tablename__ = 'engineer' - employee_id = Column(Integer, primary_key=True) - name = Column(String(50)) - engineer_info = Column(String(40)) - __mapper_args__ = {'polymorphic_identity':'engineer', - 'concrete':True} - - -The :class:`.AbstractConcreteBase` class is itself mapped, and can be -used as a target of relationships:: - - class Company(Base): - __tablename__ = 'company' - - id = Column(Integer, primary_key=True) - employees = relationship("Employee", - primaryjoin="Company.id == Employee.company_id") - - -.. versionchanged:: 0.9.3 Support for use of :class:`.AbstractConcreteBase` - as the target of a :func:`.relationship` has been improved. - -It can also be queried directly:: - - for employee in session.query(Employee).filter(Employee.name == 'qbert'): - print(employee) - - -.. _declarative_mixins: - -Mixin and Custom Base Classes -============================== - -A common need when using :mod:`~sqlalchemy.ext.declarative` is to -share some functionality, such as a set of common columns, some common -table options, or other mapped properties, across many -classes. The standard Python idioms for this is to have the classes -inherit from a base which includes these common features. - -When using :mod:`~sqlalchemy.ext.declarative`, this idiom is allowed -via the usage of a custom declarative base class, as well as a "mixin" class -which is inherited from in addition to the primary base. Declarative -includes several helper features to make this work in terms of how -mappings are declared. An example of some commonly mixed-in -idioms is below:: - - from sqlalchemy.ext.declarative import declared_attr - - class MyMixin(object): - - @declared_attr - def __tablename__(cls): - return cls.__name__.lower() - - __table_args__ = {'mysql_engine': 'InnoDB'} - __mapper_args__= {'always_refresh': True} - - id = Column(Integer, primary_key=True) - - class MyModel(MyMixin, Base): - name = Column(String(1000)) - -Where above, the class ``MyModel`` will contain an "id" column -as the primary key, a ``__tablename__`` attribute that derives -from the name of the class itself, as well as ``__table_args__`` -and ``__mapper_args__`` defined by the ``MyMixin`` mixin class. - -There's no fixed convention over whether ``MyMixin`` precedes -``Base`` or not. Normal Python method resolution rules apply, and -the above example would work just as well with:: - - class MyModel(Base, MyMixin): - name = Column(String(1000)) - -This works because ``Base`` here doesn't define any of the -variables that ``MyMixin`` defines, i.e. ``__tablename__``, -``__table_args__``, ``id``, etc. If the ``Base`` did define -an attribute of the same name, the class placed first in the -inherits list would determine which attribute is used on the -newly defined class. - -Augmenting the Base -~~~~~~~~~~~~~~~~~~~ - -In addition to using a pure mixin, most of the techniques in this -section can also be applied to the base class itself, for patterns that -should apply to all classes derived from a particular base. This is achieved -using the ``cls`` argument of the :func:`.declarative_base` function:: - - from sqlalchemy.ext.declarative import declared_attr - - class Base(object): - @declared_attr - def __tablename__(cls): - return cls.__name__.lower() - - __table_args__ = {'mysql_engine': 'InnoDB'} - - id = Column(Integer, primary_key=True) - - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base(cls=Base) - - class MyModel(Base): - name = Column(String(1000)) - -Where above, ``MyModel`` and all other classes that derive from ``Base`` will -have a table name derived from the class name, an ``id`` primary key column, -as well as the "InnoDB" engine for MySQL. - -Mixing in Columns -~~~~~~~~~~~~~~~~~ - -The most basic way to specify a column on a mixin is by simple -declaration:: - - class TimestampMixin(object): - created_at = Column(DateTime, default=func.now()) - - class MyModel(TimestampMixin, Base): - __tablename__ = 'test' - - id = Column(Integer, primary_key=True) - name = Column(String(1000)) - -Where above, all declarative classes that include ``TimestampMixin`` -will also have a column ``created_at`` that applies a timestamp to -all row insertions. - -Those familiar with the SQLAlchemy expression language know that -the object identity of clause elements defines their role in a schema. -Two ``Table`` objects ``a`` and ``b`` may both have a column called -``id``, but the way these are differentiated is that ``a.c.id`` -and ``b.c.id`` are two distinct Python objects, referencing their -parent tables ``a`` and ``b`` respectively. - -In the case of the mixin column, it seems that only one -:class:`.Column` object is explicitly created, yet the ultimate -``created_at`` column above must exist as a distinct Python object -for each separate destination class. To accomplish this, the declarative -extension creates a **copy** of each :class:`.Column` object encountered on -a class that is detected as a mixin. - -This copy mechanism is limited to simple columns that have no foreign -keys, as a :class:`.ForeignKey` itself contains references to columns -which can't be properly recreated at this level. For columns that -have foreign keys, as well as for the variety of mapper-level constructs -that require destination-explicit context, the -:class:`~.declared_attr` decorator is provided so that -patterns common to many classes can be defined as callables:: - - from sqlalchemy.ext.declarative import declared_attr - - class ReferenceAddressMixin(object): - @declared_attr - def address_id(cls): - return Column(Integer, ForeignKey('address.id')) - - class User(ReferenceAddressMixin, Base): - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - -Where above, the ``address_id`` class-level callable is executed at the -point at which the ``User`` class is constructed, and the declarative -extension can use the resulting :class:`.Column` object as returned by -the method without the need to copy it. - -.. versionchanged:: > 0.6.5 - Rename 0.6.5 ``sqlalchemy.util.classproperty`` - into :class:`~.declared_attr`. - -Columns generated by :class:`~.declared_attr` can also be -referenced by ``__mapper_args__`` to a limited degree, currently -by ``polymorphic_on`` and ``version_id_col``; the declarative extension -will resolve them at class construction time:: - - class MyMixin: - @declared_attr - def type_(cls): - return Column(String(50)) - - __mapper_args__= {'polymorphic_on':type_} - - class MyModel(MyMixin, Base): - __tablename__='test' - id = Column(Integer, primary_key=True) - - -Mixing in Relationships -~~~~~~~~~~~~~~~~~~~~~~~ - -Relationships created by :func:`~sqlalchemy.orm.relationship` are provided -with declarative mixin classes exclusively using the -:class:`.declared_attr` approach, eliminating any ambiguity -which could arise when copying a relationship and its possibly column-bound -contents. Below is an example which combines a foreign key column and a -relationship so that two classes ``Foo`` and ``Bar`` can both be configured to -reference a common target class via many-to-one:: - - class RefTargetMixin(object): - @declared_attr - def target_id(cls): - return Column('target_id', ForeignKey('target.id')) - - @declared_attr - def target(cls): - return relationship("Target") - - class Foo(RefTargetMixin, Base): - __tablename__ = 'foo' - id = Column(Integer, primary_key=True) - - class Bar(RefTargetMixin, Base): - __tablename__ = 'bar' - id = Column(Integer, primary_key=True) - - class Target(Base): - __tablename__ = 'target' - id = Column(Integer, primary_key=True) - - -Using Advanced Relationship Arguments (e.g. ``primaryjoin``, etc.) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:func:`~sqlalchemy.orm.relationship` definitions which require explicit -primaryjoin, order_by etc. expressions should in all but the most -simplistic cases use **late bound** forms -for these arguments, meaning, using either the string form or a lambda. -The reason for this is that the related :class:`.Column` objects which are to -be configured using ``@declared_attr`` are not available to another -``@declared_attr`` attribute; while the methods will work and return new -:class:`.Column` objects, those are not the :class:`.Column` objects that -Declarative will be using as it calls the methods on its own, thus using -*different* :class:`.Column` objects. - -The canonical example is the primaryjoin condition that depends upon -another mixed-in column:: - - class RefTargetMixin(object): - @declared_attr - def target_id(cls): - return Column('target_id', ForeignKey('target.id')) - - @declared_attr - def target(cls): - return relationship(Target, - primaryjoin=Target.id==cls.target_id # this is *incorrect* - ) - -Mapping a class using the above mixin, we will get an error like:: - - sqlalchemy.exc.InvalidRequestError: this ForeignKey's parent column is not - yet associated with a Table. - -This is because the ``target_id`` :class:`.Column` we've called upon in our -``target()`` method is not the same :class:`.Column` that declarative is -actually going to map to our table. - -The condition above is resolved using a lambda:: - - class RefTargetMixin(object): - @declared_attr - def target_id(cls): - return Column('target_id', ForeignKey('target.id')) - - @declared_attr - def target(cls): - return relationship(Target, - primaryjoin=lambda: Target.id==cls.target_id - ) - -or alternatively, the string form (which ultimately generates a lambda):: - - class RefTargetMixin(object): - @declared_attr - def target_id(cls): - return Column('target_id', ForeignKey('target.id')) - - @declared_attr - def target(cls): - return relationship("Target", - primaryjoin="Target.id==%s.target_id" % cls.__name__ - ) - -Mixing in deferred(), column_property(), and other MapperProperty classes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Like :func:`~sqlalchemy.orm.relationship`, all -:class:`~sqlalchemy.orm.interfaces.MapperProperty` subclasses such as -:func:`~sqlalchemy.orm.deferred`, :func:`~sqlalchemy.orm.column_property`, -etc. ultimately involve references to columns, and therefore, when -used with declarative mixins, have the :class:`.declared_attr` -requirement so that no reliance on copying is needed:: - - class SomethingMixin(object): - - @declared_attr - def dprop(cls): - return deferred(Column(Integer)) - - class Something(SomethingMixin, Base): - __tablename__ = "something" - -The :func:`.column_property` or other construct may refer -to other columns from the mixin. These are copied ahead of time before -the :class:`.declared_attr` is invoked:: - - class SomethingMixin(object): - x = Column(Integer) - - y = Column(Integer) - - @declared_attr - def x_plus_y(cls): - return column_property(cls.x + cls.y) - - -.. versionchanged:: 1.0.0 mixin columns are copied to the final mapped class - so that :class:`.declared_attr` methods can access the actual column - that will be mapped. - -Mixing in Association Proxy and Other Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Mixins can specify user-defined attributes as well as other extension -units such as :func:`.association_proxy`. The usage of -:class:`.declared_attr` is required in those cases where the attribute must -be tailored specifically to the target subclass. An example is when -constructing multiple :func:`.association_proxy` attributes which each -target a different type of child object. Below is an -:func:`.association_proxy` / mixin example which provides a scalar list of -string values to an implementing class:: - - from sqlalchemy import Column, Integer, ForeignKey, String - from sqlalchemy.orm import relationship - from sqlalchemy.ext.associationproxy import association_proxy - from sqlalchemy.ext.declarative import declarative_base, declared_attr - - Base = declarative_base() - - class HasStringCollection(object): - @declared_attr - def _strings(cls): - class StringAttribute(Base): - __tablename__ = cls.string_table_name - id = Column(Integer, primary_key=True) - value = Column(String(50), nullable=False) - parent_id = Column(Integer, - ForeignKey('%s.id' % cls.__tablename__), - nullable=False) - def __init__(self, value): - self.value = value - - return relationship(StringAttribute) - - @declared_attr - def strings(cls): - return association_proxy('_strings', 'value') - - class TypeA(HasStringCollection, Base): - __tablename__ = 'type_a' - string_table_name = 'type_a_strings' - id = Column(Integer(), primary_key=True) - - class TypeB(HasStringCollection, Base): - __tablename__ = 'type_b' - string_table_name = 'type_b_strings' - id = Column(Integer(), primary_key=True) - -Above, the ``HasStringCollection`` mixin produces a :func:`.relationship` -which refers to a newly generated class called ``StringAttribute``. The -``StringAttribute`` class is generated with its own :class:`.Table` -definition which is local to the parent class making usage of the -``HasStringCollection`` mixin. It also produces an :func:`.association_proxy` -object which proxies references to the ``strings`` attribute onto the ``value`` -attribute of each ``StringAttribute`` instance. - -``TypeA`` or ``TypeB`` can be instantiated given the constructor -argument ``strings``, a list of strings:: - - ta = TypeA(strings=['foo', 'bar']) - tb = TypeA(strings=['bat', 'bar']) - -This list will generate a collection -of ``StringAttribute`` objects, which are persisted into a table that's -local to either the ``type_a_strings`` or ``type_b_strings`` table:: - - >>> print ta._strings - [<__main__.StringAttribute object at 0x10151cd90>, - <__main__.StringAttribute object at 0x10151ce10>] - -When constructing the :func:`.association_proxy`, the -:class:`.declared_attr` decorator must be used so that a distinct -:func:`.association_proxy` object is created for each of the ``TypeA`` -and ``TypeB`` classes. - -.. versionadded:: 0.8 :class:`.declared_attr` is usable with non-mapped - attributes, including user-defined attributes as well as - :func:`.association_proxy`. - - -Controlling table inheritance with mixins -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``__tablename__`` attribute may be used to provide a function that -will determine the name of the table used for each class in an inheritance -hierarchy, as well as whether a class has its own distinct table. - -This is achieved using the :class:`.declared_attr` indicator in conjunction -with a method named ``__tablename__()``. Declarative will always -invoke :class:`.declared_attr` for the special names -``__tablename__``, ``__mapper_args__`` and ``__table_args__`` -function **for each mapped class in the hierarchy**. The function therefore -needs to expect to receive each class individually and to provide the -correct answer for each. - -For example, to create a mixin that gives every class a simple table -name based on class name:: - - from sqlalchemy.ext.declarative import declared_attr - - class Tablename: - @declared_attr - def __tablename__(cls): - return cls.__name__.lower() - - class Person(Tablename, Base): - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __tablename__ = None - __mapper_args__ = {'polymorphic_identity': 'engineer'} - primary_language = Column(String(50)) - -Alternatively, we can modify our ``__tablename__`` function to return -``None`` for subclasses, using :func:`.has_inherited_table`. This has -the effect of those subclasses being mapped with single table inheritance -agaisnt the parent:: - - from sqlalchemy.ext.declarative import declared_attr - from sqlalchemy.ext.declarative import has_inherited_table - - class Tablename(object): - @declared_attr - def __tablename__(cls): - if has_inherited_table(cls): - return None - return cls.__name__.lower() - - class Person(Tablename, Base): - id = Column(Integer, primary_key=True) - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - primary_language = Column(String(50)) - __mapper_args__ = {'polymorphic_identity': 'engineer'} - -.. _mixin_inheritance_columns: - -Mixing in Columns in Inheritance Scenarios -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In constrast to how ``__tablename__`` and other special names are handled when -used with :class:`.declared_attr`, when we mix in columns and properties (e.g. -relationships, column properties, etc.), the function is -invoked for the **base class only** in the hierarchy. Below, only the -``Person`` class will receive a column -called ``id``; the mapping will fail on ``Engineer``, which is not given -a primary key:: - - class HasId(object): - @declared_attr - def id(cls): - return Column('id', Integer, primary_key=True) - - class Person(HasId, Base): - __tablename__ = 'person' - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __tablename__ = 'engineer' - primary_language = Column(String(50)) - __mapper_args__ = {'polymorphic_identity': 'engineer'} - -It is usually the case in joined-table inheritance that we want distinctly -named columns on each subclass. However in this case, we may want to have -an ``id`` column on every table, and have them refer to each other via -foreign key. We can achieve this as a mixin by using the -:attr:`.declared_attr.cascading` modifier, which indicates that the -function should be invoked **for each class in the hierarchy**, just like -it does for ``__tablename__``:: - - class HasId(object): - @declared_attr.cascading - def id(cls): - if has_inherited_table(cls): - return Column('id', - Integer, - ForeignKey('person.id'), primary_key=True) - else: - return Column('id', Integer, primary_key=True) - - class Person(HasId, Base): - __tablename__ = 'person' - discriminator = Column('type', String(50)) - __mapper_args__ = {'polymorphic_on': discriminator} - - class Engineer(Person): - __tablename__ = 'engineer' - primary_language = Column(String(50)) - __mapper_args__ = {'polymorphic_identity': 'engineer'} - - -.. versionadded:: 1.0.0 added :attr:`.declared_attr.cascading`. - -Combining Table/Mapper Arguments from Multiple Mixins -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the case of ``__table_args__`` or ``__mapper_args__`` -specified with declarative mixins, you may want to combine -some parameters from several mixins with those you wish to -define on the class iteself. The -:class:`.declared_attr` decorator can be used -here to create user-defined collation routines that pull -from multiple collections:: - - from sqlalchemy.ext.declarative import declared_attr - - class MySQLSettings(object): - __table_args__ = {'mysql_engine':'InnoDB'} - - class MyOtherMixin(object): - __table_args__ = {'info':'foo'} - - class MyModel(MySQLSettings, MyOtherMixin, Base): - __tablename__='my_model' - - @declared_attr - def __table_args__(cls): - args = dict() - args.update(MySQLSettings.__table_args__) - args.update(MyOtherMixin.__table_args__) - return args - - id = Column(Integer, primary_key=True) - -Creating Indexes with Mixins -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To define a named, potentially multicolumn :class:`.Index` that applies to all -tables derived from a mixin, use the "inline" form of :class:`.Index` and -establish it as part of ``__table_args__``:: - - class MyMixin(object): - a = Column(Integer) - b = Column(Integer) - - @declared_attr - def __table_args__(cls): - return (Index('test_idx_%s' % cls.__tablename__, 'a', 'b'),) - - class MyModel(MyMixin, Base): - __tablename__ = 'atable' - c = Column(Integer,primary_key=True) - -Special Directives -================== - -``__declare_last__()`` -~~~~~~~~~~~~~~~~~~~~~~ - -The ``__declare_last__()`` hook allows definition of -a class level function that is automatically called by the -:meth:`.MapperEvents.after_configured` event, which occurs after mappings are -assumed to be completed and the 'configure' step has finished:: - - class MyClass(Base): - @classmethod - def __declare_last__(cls): - "" - # do something with mappings - -.. versionadded:: 0.7.3 - -``__declare_first__()`` -~~~~~~~~~~~~~~~~~~~~~~~ - -Like ``__declare_last__()``, but is called at the beginning of mapper -configuration via the :meth:`.MapperEvents.before_configured` event:: - - class MyClass(Base): - @classmethod - def __declare_first__(cls): - "" - # do something before mappings are configured - -.. versionadded:: 0.9.3 - -.. _declarative_abstract: - -``__abstract__`` -~~~~~~~~~~~~~~~~~~~ - -``__abstract__`` causes declarative to skip the production -of a table or mapper for the class entirely. A class can be added within a -hierarchy in the same way as mixin (see :ref:`declarative_mixins`), allowing -subclasses to extend just from the special class:: - - class SomeAbstractBase(Base): - __abstract__ = True - - def some_helpful_method(self): - "" - - @declared_attr - def __mapper_args__(cls): - return {"helpful mapper arguments":True} - - class MyMappedClass(SomeAbstractBase): - "" - -One possible use of ``__abstract__`` is to use a distinct -:class:`.MetaData` for different bases:: - - Base = declarative_base() - - class DefaultBase(Base): - __abstract__ = True - metadata = MetaData() - - class OtherBase(Base): - __abstract__ = True - metadata = MetaData() - -Above, classes which inherit from ``DefaultBase`` will use one -:class:`.MetaData` as the registry of tables, and those which inherit from -``OtherBase`` will use a different one. The tables themselves can then be -created perhaps within distinct databases:: - - DefaultBase.metadata.create_all(some_engine) - OtherBase.metadata_create_all(some_other_engine) - -.. versionadded:: 0.7.3 - -Class Constructor -================= - -As a convenience feature, the :func:`declarative_base` sets a default -constructor on classes which takes keyword arguments, and assigns them -to the named attributes:: - - e = Engineer(primary_language='python') - -Sessions -======== - -Note that ``declarative`` does nothing special with sessions, and is -only intended as an easier way to configure mappers and -:class:`~sqlalchemy.schema.Table` objects. A typical application -setup using :class:`~sqlalchemy.orm.scoping.scoped_session` might look like:: - - engine = create_engine('postgresql://scott:tiger@localhost/test') - Session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=engine)) - Base = declarative_base() - -Mapped instances then make usage of -:class:`~sqlalchemy.orm.session.Session` in the usual way. - -""" - from .api import declarative_base, synonym_for, comparable_using, \ instrument_declarative, ConcreteBase, AbstractConcreteBase, \ DeclarativeMeta, DeferredReflection, has_inherited_table,\ -- cgit v1.2.1 From 023067a0b759abc7e72363116ba64b29d8a4d15b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 17 Dec 2014 19:20:06 -0500 Subject: - classical is really not the most important topic here --- doc/build/orm/mapper_config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/orm/mapper_config.rst b/doc/build/orm/mapper_config.rst index b68c2331d..671abdfd6 100644 --- a/doc/build/orm/mapper_config.rst +++ b/doc/build/orm/mapper_config.rst @@ -13,9 +13,9 @@ know how to construct and use rudimentary mappers and relationships. .. toctree:: :maxdepth: 2 - classical scalar_mapping inheritance nonstandard_mappings + classical versioning mapping_api \ No newline at end of file -- cgit v1.2.1 From 0d15791a6e3ec35e3c2e297026d8396742e30a34 Mon Sep 17 00:00:00 2001 From: Rob Berry Date: Thu, 18 Dec 2014 13:17:11 +0000 Subject: Update gaerdbms to highlight improved connection method --- lib/sqlalchemy/dialects/mysql/gaerdbms.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/sqlalchemy/dialects/mysql/gaerdbms.py b/lib/sqlalchemy/dialects/mysql/gaerdbms.py index 0059f5a65..947b066cc 100644 --- a/lib/sqlalchemy/dialects/mysql/gaerdbms.py +++ b/lib/sqlalchemy/dialects/mysql/gaerdbms.py @@ -17,6 +17,10 @@ developers-guide .. versionadded:: 0.7.8 + .. deprecated:: 1.0 Cloud SQL now recommends creating connections via the + mysql dialect using the URL format + `mysql://root@/?unix_socket=/cloudsql/:` + Pooling ------- @@ -33,6 +37,14 @@ import os from .mysqldb import MySQLDialect_mysqldb from ...pool import NullPool import re +from sqlalchemy.util import warn_deprecated + + +warn_deprecated( + "Google Cloud SQL now recommends creating connections via the " + "mysql dialect using the URL format " + "mysql://root@/?unix_socket=/cloudsql/:" +) def _is_dev_environment(): -- cgit v1.2.1 From b92589e3a0b2bc52d842b6b543e5976b694cc214 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 19 Dec 2014 11:55:10 -0500 Subject: - create a new section on "custom load rules", to help with edge cases like that of #3277. fixes #3277 --- doc/build/orm/loading_relationships.rst | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/doc/build/orm/loading_relationships.rst b/doc/build/orm/loading_relationships.rst index b2d8124e2..297392f3e 100644 --- a/doc/build/orm/loading_relationships.rst +++ b/doc/build/orm/loading_relationships.rst @@ -516,6 +516,82 @@ to a string SQL statement:: "from users left outer join addresses on users.user_id=addresses.user_id").\ options(contains_eager(User.addresses, alias=eager_columns)) +Creating Custom Load Rules +--------------------------- + +.. warning:: This is an advanced technique! Great care and testing + should be applied. + +The ORM has various edge cases where the value of an attribute is locally +available, however the ORM itself doesn't have awareness of this. There +are also cases when a user-defined system of loading attributes is desirable. +To support the use case of user-defined loading systems, a key function +:func:`.attributes.set_committed_value` is provided. This function is +basically equivalent to Python's own ``setattr()`` function, except that +when applied to a target object, SQLAlchemy's "attribute history" system +which is used to determine flush-time changes is bypassed; the attribute +is assigned in the same way as if the ORM loaded it that way from the database. + +The use of :func:`.attributes.set_committed_value` can be combined with another +key event known as :meth:`.InstanceEvents.load` to produce attribute-population +behaviors when an object is loaded. One such example is the bi-directional +"one-to-one" case, where loading the "many-to-one" side of a one-to-one +should also imply the value of the "one-to-many" side. The SQLAlchemy ORM +does not consider backrefs when loading related objects, and it views a +"one-to-one" as just another "one-to-many", that just happens to be one +row. + +Given the following mapping:: + + from sqlalchemy import Integer, ForeignKey, Column + from sqlalchemy.orm import relationship, backref + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + + class A(Base): + __tablename__ = 'a' + id = Column(Integer, primary_key=True) + b_id = Column(ForeignKey('b.id')) + b = relationship("B", backref=backref("a", uselist=False), lazy='joined') + + + class B(Base): + __tablename__ = 'b' + id = Column(Integer, primary_key=True) + + +If we query for an ``A`` row, and then ask it for ``a.b.a``, we will get +an extra SELECT:: + + >>> a1.b.a + SELECT a.id AS a_id, a.b_id AS a_b_id + FROM a + WHERE ? = a.b_id + +This SELECT is redundant becasue ``b.a`` is the same value as ``a1``. We +can create an on-load rule to populate this for us:: + + from sqlalchemy import event + from sqlalchemy.orm import attributes + + @event.listens_for(A, "load") + def load_b(target, context): + if 'b' in target.__dict__: + attributes.set_committed_value(target.b, 'a', target) + +Now when we query for ``A``, we will get ``A.b`` from the joined eager load, +and ``A.b.a`` from our event: + +.. sourcecode:: pycon+sql + + {sql}a1 = s.query(A).first() + SELECT a.id AS a_id, a.b_id AS a_b_id, b_1.id AS b_1_id + FROM a LEFT OUTER JOIN b AS b_1 ON b_1.id = a.b_id + LIMIT ? OFFSET ? + (1, 0) + {stop}assert a1.b.a is a1 Relationship Loader API -- cgit v1.2.1 From d1ac6cb33af3b105db7cdb51411e10ac3bafff1f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 19 Dec 2014 12:14:52 -0500 Subject: - Fixed bug where using a :class:`.TypeDecorator` that implemented a type that was also a :class:`.TypeDecorator` would fail with Python's "Cannot create a consistent method resolution order (MRO)" error, when any kind of SQL comparison expression were used against an object using this type. --- doc/build/changelog/changelog_09.rst | 11 +++++++++++ lib/sqlalchemy/sql/type_api.py | 10 +++++++--- test/sql/test_operators.py | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index b2c876141..4ff73c45d 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,17 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, sql + :versions: 1.0.0 + :tickets: 3278 + + Fixed bug where using a :class:`.TypeDecorator` that implemented + a type that was also a :class:`.TypeDecorator` would fail with + Python's "Cannot create a consistent method resolution order (MRO)" + error, when any kind of SQL comparison expression were used against + an object using this type. + .. change:: :tags: bug, mysql :versions: 1.0.0 diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py index d3e0a008e..d414daf2a 100644 --- a/lib/sqlalchemy/sql/type_api.py +++ b/lib/sqlalchemy/sql/type_api.py @@ -630,9 +630,13 @@ class TypeDecorator(TypeEngine): @property def comparator_factory(self): - return type("TDComparator", - (TypeDecorator.Comparator, self.impl.comparator_factory), - {}) + if TypeDecorator.Comparator in self.impl.comparator_factory.__mro__: + return self.impl.comparator_factory + else: + return type("TDComparator", + (TypeDecorator.Comparator, + self.impl.comparator_factory), + {}) def _gen_dialect_impl(self, dialect): """ diff --git a/test/sql/test_operators.py b/test/sql/test_operators.py index f8ac1528f..3b8b20513 100644 --- a/test/sql/test_operators.py +++ b/test/sql/test_operators.py @@ -393,6 +393,31 @@ class TypeDecoratorComparatorTest(_CustomComparatorTests, fixtures.TestBase): return MyInteger +class TypeDecoratorTypeDecoratorComparatorTest( + _CustomComparatorTests, fixtures.TestBase): + + def _add_override_factory(self): + + class MyIntegerOne(TypeDecorator): + impl = Integer + + class comparator_factory(TypeDecorator.Comparator): + + def __init__(self, expr): + self.expr = expr + + def __add__(self, other): + return self.expr.op("goofy")(other) + + def __and__(self, other): + return self.expr.op("goofy_and")(other) + + class MyIntegerTwo(TypeDecorator): + impl = MyIntegerOne + + return MyIntegerTwo + + class TypeDecoratorWVariantComparatorTest( _CustomComparatorTests, fixtures.TestBase): -- cgit v1.2.1 From 8ae47dc6e0a98b359247040236be0810b9086f40 Mon Sep 17 00:00:00 2001 From: Priit Laes Date: Fri, 19 Dec 2014 18:46:16 +0200 Subject: Maul the evaulate & friends typo --- doc/build/changelog/changelog_07.rst | 2 +- doc/build/changelog/changelog_08.rst | 2 +- doc/build/changelog/changelog_09.rst | 4 ++-- doc/build/changelog/changelog_10.rst | 2 +- doc/build/changelog/migration_10.rst | 2 +- examples/sharding/attribute_shard.py | 2 +- lib/sqlalchemy/dialects/postgresql/json.py | 2 +- lib/sqlalchemy/ext/declarative/base.py | 2 +- lib/sqlalchemy/orm/query.py | 4 ++-- lib/sqlalchemy/sql/dml.py | 2 +- lib/sqlalchemy/sql/elements.py | 2 +- lib/sqlalchemy/sql/operators.py | 2 +- lib/sqlalchemy/util/langhelpers.py | 2 +- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/build/changelog/changelog_07.rst b/doc/build/changelog/changelog_07.rst index 5504a0ad6..e782ba938 100644 --- a/doc/build/changelog/changelog_07.rst +++ b/doc/build/changelog/changelog_07.rst @@ -3517,7 +3517,7 @@ :tags: orm :tickets: 2122 - Some fixes to "evaulate" and "fetch" evaluation + Some fixes to "evaluate" and "fetch" evaluation when query.update(), query.delete() are called. The retrieval of records is done after autoflush in all cases, and before update/delete is diff --git a/doc/build/changelog/changelog_08.rst b/doc/build/changelog/changelog_08.rst index 6515f731d..baaa7b15b 100644 --- a/doc/build/changelog/changelog_08.rst +++ b/doc/build/changelog/changelog_08.rst @@ -2214,7 +2214,7 @@ expr1 = mycolumn > 2 bool(expr1 == expr1) - Would evaulate as ``False``, even though this is an identity + Would evaluate as ``False``, even though this is an identity comparison, because ``mycolumn > 2`` would be "grouped" before being placed into the :class:`.BinaryExpression`, thus changing its identity. :class:`.BinaryExpression` now keeps track diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 4ff73c45d..f92d8324b 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -548,7 +548,7 @@ :tags: bug, orm :tickets: 3117 - The "evaulator" for query.update()/delete() won't work with multi-table + The "evaluator" for query.update()/delete() won't work with multi-table updates, and needs to be set to `synchronize_session=False` or `synchronize_session='fetch'`; a warning is now emitted. In 1.0 this will be promoted to a full exception. @@ -2915,7 +2915,7 @@ in an ``ORDER BY`` clause, if that label is also referred to in the columns clause of the select, instead of rewriting the full expression. This gives the database a better chance to - optimize the evaulation of the same expression in two different + optimize the evaluation of the same expression in two different contexts. .. seealso:: diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 4da7b9456..efd9d51d6 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -959,7 +959,7 @@ :tags: bug, orm :tickets: 3117 - The "evaulator" for query.update()/delete() won't work with multi-table + The "evaluator" for query.update()/delete() won't work with multi-table updates, and needs to be set to `synchronize_session=False` or `synchronize_session='fetch'`; this now raises an exception, with a message to change the synchronize setting. diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index db0d270a1..717f31aff 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1010,7 +1010,7 @@ join into a subquery as a join target on SQLite. query.update() with ``synchronize_session='evaluate'`` raises on multi-table update ----------------------------------------------------------------------------------- -The "evaulator" for :meth:`.Query.update` won't work with multi-table +The "evaluator" for :meth:`.Query.update` won't work with multi-table updates, and needs to be set to ``synchronize_session=False`` or ``synchronize_session='fetch'`` when multiple tables are present. The new behavior is that an explicit exception is now raised, with a message diff --git a/examples/sharding/attribute_shard.py b/examples/sharding/attribute_shard.py index 34b1be5b2..4ce8c247f 100644 --- a/examples/sharding/attribute_shard.py +++ b/examples/sharding/attribute_shard.py @@ -168,7 +168,7 @@ def _get_query_comparisons(query): elif bind.callable: # some ORM functions (lazy loading) # place the bind's value as a - # callable for deferred evaulation. + # callable for deferred evaluation. value = bind.callable() else: # just use .value diff --git a/lib/sqlalchemy/dialects/postgresql/json.py b/lib/sqlalchemy/dialects/postgresql/json.py index 250bf5e9d..50176918e 100644 --- a/lib/sqlalchemy/dialects/postgresql/json.py +++ b/lib/sqlalchemy/dialects/postgresql/json.py @@ -77,7 +77,7 @@ class JSONElement(elements.BinaryExpression): def cast(self, type_): """Convert this :class:`.JSONElement` to apply both the 'astext' operator - as well as an explicit type cast when evaulated. + as well as an explicit type cast when evaluated. E.g.:: diff --git a/lib/sqlalchemy/ext/declarative/base.py b/lib/sqlalchemy/ext/declarative/base.py index 291608b6c..6735abf4c 100644 --- a/lib/sqlalchemy/ext/declarative/base.py +++ b/lib/sqlalchemy/ext/declarative/base.py @@ -278,7 +278,7 @@ class _MapperConfig(object): elif not isinstance(value, (Column, MapperProperty)): # using @declared_attr for some object that # isn't Column/MapperProperty; remove from the dict_ - # and place the evaulated value onto the class. + # and place the evaluated value onto the class. if not k.startswith('__'): dict_.pop(k) setattr(cls, k, value) diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 1afffb90e..6cade322a 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -2788,7 +2788,7 @@ class Query(object): SELECT statement emitted and will significantly reduce performance. - * The ``'evaulate'`` strategy performs a scan of + * The ``'evaluate'`` strategy performs a scan of all matching objects within the :class:`.Session`; if the contents of the :class:`.Session` are expired, such as via a proceeding :meth:`.Session.commit` call, **this will @@ -2885,7 +2885,7 @@ class Query(object): SELECT statement emitted and will significantly reduce performance. - * The ``'evaulate'`` strategy performs a scan of + * The ``'evaluate'`` strategy performs a scan of all matching objects within the :class:`.Session`; if the contents of the :class:`.Session` are expired, such as via a proceeding :meth:`.Session.commit` call, **this will diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py index 9f2ce7ce3..62169319b 100644 --- a/lib/sqlalchemy/sql/dml.py +++ b/lib/sqlalchemy/sql/dml.py @@ -387,7 +387,7 @@ class ValuesBase(UpdateBase): :func:`.mapper`. :param cols: optional list of column key names or :class:`.Column` - objects. If omitted, all column expressions evaulated on the server + objects. If omitted, all column expressions evaluated on the server are added to the returning list. .. versionadded:: 0.9.0 diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index 30965c801..0d49041c7 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -2146,7 +2146,7 @@ class Case(ColumnElement): result of the ``CASE`` construct if all expressions within :paramref:`.case.whens` evaluate to false. When omitted, most databases will produce a result of NULL if none of the "when" - expressions evaulate to true. + expressions evaluate to true. """ diff --git a/lib/sqlalchemy/sql/operators.py b/lib/sqlalchemy/sql/operators.py index b08e44ab8..a328b023e 100644 --- a/lib/sqlalchemy/sql/operators.py +++ b/lib/sqlalchemy/sql/operators.py @@ -137,7 +137,7 @@ class Operators(object): .. versionadded:: 0.8 - added the 'precedence' argument. :param is_comparison: if True, the operator will be considered as a - "comparison" operator, that is which evaulates to a boolean + "comparison" operator, that is which evaluates to a boolean true/false value, like ``==``, ``>``, etc. This flag should be set so that ORM relationships can establish that the operator is a comparison operator when used in a custom join condition. diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 5c17bea88..ac6b50de2 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -936,7 +936,7 @@ def asbool(obj): def bool_or_str(*text): - """Return a callable that will evaulate a string as + """Return a callable that will evaluate a string as boolean, or one of a set of "alternate" string values. """ -- cgit v1.2.1 From 182553b7409cfa6673483d03f29bc4c462336577 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 19 Dec 2014 14:15:56 -0500 Subject: - make the google deprecation messages more specific, use full URL format - add an extra doc to MySQLdb - changelog --- doc/build/changelog/changelog_09.rst | 9 +++++++++ lib/sqlalchemy/dialects/mysql/gaerdbms.py | 11 +++++++---- lib/sqlalchemy/dialects/mysql/mysqldb.py | 8 ++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index f92d8324b..7505ee50c 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,15 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: change, mysql + :versions: 1.0.0 + :tickets: 3275 + + The ``gaerdbms`` dialect is no longer necessary, and emits a + deprecation warning. Google now recommends using the MySQLdb + dialect directly. + .. change:: :tags: bug, sql :versions: 1.0.0 diff --git a/lib/sqlalchemy/dialects/mysql/gaerdbms.py b/lib/sqlalchemy/dialects/mysql/gaerdbms.py index 947b066cc..b4daec69c 100644 --- a/lib/sqlalchemy/dialects/mysql/gaerdbms.py +++ b/lib/sqlalchemy/dialects/mysql/gaerdbms.py @@ -17,9 +17,12 @@ developers-guide .. versionadded:: 0.7.8 - .. deprecated:: 1.0 Cloud SQL now recommends creating connections via the + .. deprecated:: 1.0 This dialect is **no longer necessary** for + Google Cloud SQL; the MySQLdb dialect can be used directly. + Cloud SQL now recommends creating connections via the mysql dialect using the URL format - `mysql://root@/?unix_socket=/cloudsql/:` + + `mysql+mysqldb://root@/?unix_socket=/cloudsql/:` Pooling @@ -42,8 +45,8 @@ from sqlalchemy.util import warn_deprecated warn_deprecated( "Google Cloud SQL now recommends creating connections via the " - "mysql dialect using the URL format " - "mysql://root@/?unix_socket=/cloudsql/:" + "MySQLdb dialect directly, using the URL format " + "mysql+mysqldb://root@/?unix_socket=/cloudsql/:" ) diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index 5bb67a24d..929317467 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -39,6 +39,14 @@ MySQL-python version 1.2.2 has a serious memory leak related to unicode conversion, a feature which is disabled via ``use_unicode=0``. It is strongly advised to use the latest version of MySQL-Python. +Using MySQLdb with Google Cloud SQL +----------------------------------- + +Google Cloud SQL now recommends use of the MySQLdb dialect. Connect +using a URL like the following:: + + mysql+mysqldb://root@/?unix_socket=/cloudsql/: + """ from .base import (MySQLDialect, MySQLExecutionContext, -- cgit v1.2.1 From 5659ecb2e8a4aac83a1eb9b2c5ea348f0077ca72 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 19 Dec 2014 18:20:11 -0500 Subject: - ouch, this needs to be in dbapi, not module level! --- lib/sqlalchemy/dialects/mysql/gaerdbms.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/sqlalchemy/dialects/mysql/gaerdbms.py b/lib/sqlalchemy/dialects/mysql/gaerdbms.py index b4daec69c..284b51bde 100644 --- a/lib/sqlalchemy/dialects/mysql/gaerdbms.py +++ b/lib/sqlalchemy/dialects/mysql/gaerdbms.py @@ -43,13 +43,6 @@ import re from sqlalchemy.util import warn_deprecated -warn_deprecated( - "Google Cloud SQL now recommends creating connections via the " - "MySQLdb dialect directly, using the URL format " - "mysql+mysqldb://root@/?unix_socket=/cloudsql/:" -) - - def _is_dev_environment(): return os.environ.get('SERVER_SOFTWARE', '').startswith('Development/') @@ -58,6 +51,14 @@ class MySQLDialect_gaerdbms(MySQLDialect_mysqldb): @classmethod def dbapi(cls): + + warn_deprecated( + "Google Cloud SQL now recommends creating connections via the " + "MySQLdb dialect directly, using the URL format " + "mysql+mysqldb://root@/?unix_socket=/cloudsql/" + ":" + ) + # from django: # http://code.google.com/p/googleappengine/source/ # browse/trunk/python/google/storage/speckle/ -- cgit v1.2.1 From ef6dc0cf2ef581e7cb53dcb4840f678aa1fa5ba6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 12:47:57 -0500 Subject: - typo fixes #3269 --- lib/sqlalchemy/sql/elements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index 0d49041c7..445857b82 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -1279,7 +1279,7 @@ class TextClause(Executable, ClauseElement): E.g.:: - fom sqlalchemy import text + from sqlalchemy import text t = text("SELECT * FROM users") result = connection.execute(t) -- cgit v1.2.1 From 544e72bcb6af1ca657b1762f105634372eca3bc0 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 15:55:30 -0500 Subject: - corrections - attempt to add a script to semi-automate the fixing of links --- doc/build/changelog/changelog_09.rst | 15 +++++----- doc/build/changelog/changelog_10.rst | 6 ++-- doc/build/changelog/migration_10.rst | 6 ++-- doc/build/conf.py | 3 ++ doc/build/core/compiler.rst | 2 +- doc/build/core/custom_types.rst | 2 ++ doc/build/core/exceptions.rst | 2 +- doc/build/core/sqla_engine_arch.png | Bin 28189 -> 28190 bytes doc/build/core/types.rst | 2 +- doc/build/corrections.py | 39 +++++++++++++++++++++++++ doc/build/dialects/sqlite.rst | 2 +- doc/build/orm/collections.rst | 2 +- doc/build/orm/constructors.rst | 2 ++ doc/build/orm/exceptions.rst | 2 +- doc/build/orm/extensions/associationproxy.rst | 2 +- doc/build/orm/extensions/mutable.rst | 2 +- doc/build/orm/internals.rst | 6 ++++ doc/build/orm/join_conditions.rst | 2 +- doc/build/orm/mapper_config.rst | 2 +- doc/build/orm/session_api.rst | 2 ++ lib/sqlalchemy/dialects/postgresql/base.py | 4 +-- lib/sqlalchemy/dialects/postgresql/psycopg2.py | 9 +++--- lib/sqlalchemy/orm/session.py | 4 +-- lib/sqlalchemy/pool.py | 6 ++-- lib/sqlalchemy/sql/expression.py | 2 +- lib/sqlalchemy/sql/schema.py | 8 +++++ lib/sqlalchemy/sql/sqltypes.py | 2 +- 27 files changed, 99 insertions(+), 37 deletions(-) create mode 100644 doc/build/corrections.py diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 7505ee50c..e0f46eb66 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -1,3 +1,4 @@ + ============== 0.9 Changelog ============== @@ -314,7 +315,7 @@ :versions: 1.0.0 :pullrequest: bitbucket:28 - Fixed bug where :ref:`ext.mutable.MutableDict` + Fixed bug where :class:`.ext.mutable.MutableDict` failed to implement the ``update()`` dictionary method, thus not catching changes. Pull request courtesy Matt Chisholm. @@ -323,9 +324,9 @@ :versions: 1.0.0 :pullrequest: bitbucket:27 - Fixed bug where a custom subclass of :ref:`ext.mutable.MutableDict` + Fixed bug where a custom subclass of :class:`.ext.mutable.MutableDict` would not show up in a "coerce" operation, and would instead - return a plain :ref:`ext.mutable.MutableDict`. Pull request + return a plain :class:`.ext.mutable.MutableDict`. Pull request courtesy Matt Chisholm. .. change:: @@ -577,7 +578,7 @@ :tickets: 3078 Added kw argument ``postgresql_regconfig`` to the - :meth:`.Operators.match` operator, allows the "reg config" argument + :meth:`.ColumnOperators.match` operator, allows the "reg config" argument to be specified to the ``to_tsquery()`` function emitted. Pull request courtesy Jonathan Vanasco. @@ -866,7 +867,7 @@ translated through some kind of SQL function or expression. This is kind of experimental, but the first proof of concept is a "materialized path" join condition where a path string is compared - to itself using "like". The :meth:`.Operators.like` operator has + to itself using "like". The :meth:`.ColumnOperators.like` operator has also been added to the list of valid operators to use in a primaryjoin condition. @@ -1939,8 +1940,8 @@ Fixed an issue where the C extensions in Py3K are using the wrong API to specify the top-level module function, which breaks in Python 3.4b2. Py3.4b2 changes PyMODINIT_FUNC to return - "void" instead of "PyObject *", so we now make sure to use - "PyMODINIT_FUNC" instead of "PyObject *" directly. Pull request + "void" instead of ``PyObject *``, so we now make sure to use + "PyMODINIT_FUNC" instead of ``PyObject *`` directly. Pull request courtesy cgohlke. .. change:: diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index efd9d51d6..ceed4d912 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -147,7 +147,7 @@ :tags: bug, mysql :tickets: 3263 - The :meth:`.Operators.match` operator is now handled such that the + The :meth:`.ColumnOperators.match` operator is now handled such that the return type is not strictly assumed to be boolean; it now returns a :class:`.Boolean` subclass called :class:`.MatchType`. The type will still produce boolean behavior when used in Python @@ -861,7 +861,7 @@ .. change:: :tags: bug, orm, py3k - The :class:`.IdentityMap` exposed from :class:`.Session.identity` + The :class:`.IdentityMap` exposed from :class:`.Session.identity_map` now returns lists for ``items()`` and ``values()`` in Py3K. Early porting to Py3K here had these returning iterators, when they technically should be "iterable views"..for now, lists are OK. @@ -911,7 +911,7 @@ :tags: orm, feature :tickets: 2971 - The :meth:`.InspectionAttr.info` collection is now moved down to + The :attr:`.InspectionAttr.info` collection is now moved down to :class:`.InspectionAttr`, where in addition to being available on all :class:`.MapperProperty` objects, it is also now available on hybrid properties, association proxies, when accessed via diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 717f31aff..829d04c51 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -905,7 +905,7 @@ as all the subclasses normally refer to the same table:: -.. _migration_migration_deprecated_orm_events: +.. _migration_deprecated_orm_events: Deprecated ORM Event Hooks Removed ---------------------------------- @@ -1624,7 +1624,7 @@ again works on MySQL. The match() operator now returns an agnostic MatchType compatible with MySQL's floating point return value ---------------------------------------------------------------------------------------------------------- -The return type of a :meth:`.Operators.match` expression is now a new type +The return type of a :meth:`.ColumnOperators.match` expression is now a new type called :class:`.MatchType`. This is a subclass of :class:`.Boolean`, that can be intercepted by the dialect in order to produce a different result type at SQL execution time. @@ -1669,8 +1669,6 @@ on polishing it. Dialect Improvements and Changes - SQLite ============================================= -.. _change_2984: - SQLite named and unnamed UNIQUE and FOREIGN KEY constraints will inspect and reflect ------------------------------------------------------------------------------------- diff --git a/doc/build/conf.py b/doc/build/conf.py index 7e17fcd59..02784bdae 100644 --- a/doc/build/conf.py +++ b/doc/build/conf.py @@ -37,6 +37,7 @@ extensions = [ 'zzzeeksphinx', 'changelog', 'sphinx_paramlinks', + 'corrections' ] # Add any paths that contain templates here, relative to this directory. @@ -341,3 +342,5 @@ intersphinx_mapping = { 'alembic': ('http://alembic.readthedocs.org/en/latest/', None), 'psycopg2': ('http://pythonhosted.org/psycopg2', None), } + + diff --git a/doc/build/core/compiler.rst b/doc/build/core/compiler.rst index 73c9e3995..202ef2b0e 100644 --- a/doc/build/core/compiler.rst +++ b/doc/build/core/compiler.rst @@ -4,4 +4,4 @@ Custom SQL Constructs and Compilation Extension =============================================== .. automodule:: sqlalchemy.ext.compiler - :members: \ No newline at end of file + :members: diff --git a/doc/build/core/custom_types.rst b/doc/build/core/custom_types.rst index 92c5ca6cf..8d0c42703 100644 --- a/doc/build/core/custom_types.rst +++ b/doc/build/core/custom_types.rst @@ -1,3 +1,5 @@ +.. module:: sqlalchemy.types + .. _types_custom: Custom Types diff --git a/doc/build/core/exceptions.rst b/doc/build/core/exceptions.rst index 30270f8b0..63bbc1e15 100644 --- a/doc/build/core/exceptions.rst +++ b/doc/build/core/exceptions.rst @@ -2,4 +2,4 @@ Core Exceptions =============== .. automodule:: sqlalchemy.exc - :members: \ No newline at end of file + :members: diff --git a/doc/build/core/sqla_engine_arch.png b/doc/build/core/sqla_engine_arch.png index f54d105bd..f040a2cf3 100644 Binary files a/doc/build/core/sqla_engine_arch.png and b/doc/build/core/sqla_engine_arch.png differ diff --git a/doc/build/core/types.rst b/doc/build/core/types.rst index 9d2b66124..ab761a1cb 100644 --- a/doc/build/core/types.rst +++ b/doc/build/core/types.rst @@ -8,4 +8,4 @@ Column and Data Types type_basics custom_types - type_api \ No newline at end of file + type_api diff --git a/doc/build/corrections.py b/doc/build/corrections.py new file mode 100644 index 000000000..fa2e13a38 --- /dev/null +++ b/doc/build/corrections.py @@ -0,0 +1,39 @@ +targets = {} +quit = False +def missing_reference(app, env, node, contnode): + global quit + if quit: + return + reftarget = node.attributes['reftarget'] + reftype = node.attributes['reftype'] + refdoc = node.attributes['refdoc'] + rawsource = node.rawsource + if reftype == 'paramref': + return + + target = rawsource + if target in targets: + return + print "\n%s" % refdoc + print "Reftarget: %s" % rawsource + correction = raw_input("? ") + correction = correction.strip() + if correction == ".": + correction = ":%s:`.%s`" % (reftype, reftarget) + elif correction == 'q': + quit = True + else: + targets[target] = correction + +def write_corrections(app, exception): + print "#!/bin/sh\n\n" + for targ, corr in targets.items(): + if not corr: + continue + + print """find lib/ -print -type f -name "*.py" -exec sed -i '' 's/%s/%s/g' {} \;""" % (targ, corr) + print """find doc/build/ -print -type f -name "*.rst" -exec sed -i '' 's/%s/%s/g' {} \;""" % (targ, corr) + +def setup(app): + app.connect('missing-reference', missing_reference) + app.connect('build-finished', write_corrections) diff --git a/doc/build/dialects/sqlite.rst b/doc/build/dialects/sqlite.rst index a18b0ba7b..93a54ee8d 100644 --- a/doc/build/dialects/sqlite.rst +++ b/doc/build/dialects/sqlite.rst @@ -33,4 +33,4 @@ Pysqlite Pysqlcipher ----------- -.. automodule:: sqlalchemy.dialects.sqlite.pysqlcipher \ No newline at end of file +.. automodule:: sqlalchemy.dialects.sqlite.pysqlcipher diff --git a/doc/build/orm/collections.rst b/doc/build/orm/collections.rst index 898f70ebb..7d474ce65 100644 --- a/doc/build/orm/collections.rst +++ b/doc/build/orm/collections.rst @@ -573,7 +573,7 @@ Various internal methods. .. autoclass:: collection -.. autofunction:: collection_adapter +.. autodata:: collection_adapter .. autoclass:: CollectionAdapter diff --git a/doc/build/orm/constructors.rst b/doc/build/orm/constructors.rst index ab6691553..38cbb4182 100644 --- a/doc/build/orm/constructors.rst +++ b/doc/build/orm/constructors.rst @@ -1,3 +1,5 @@ +.. module:: sqlalchemy.orm + .. _mapping_constructors: Constructors and Object Initialization diff --git a/doc/build/orm/exceptions.rst b/doc/build/orm/exceptions.rst index f95b26eed..047c743e0 100644 --- a/doc/build/orm/exceptions.rst +++ b/doc/build/orm/exceptions.rst @@ -2,4 +2,4 @@ ORM Exceptions ============== .. automodule:: sqlalchemy.orm.exc - :members: \ No newline at end of file + :members: diff --git a/doc/build/orm/extensions/associationproxy.rst b/doc/build/orm/extensions/associationproxy.rst index 9b25c4a68..6fc57e30c 100644 --- a/doc/build/orm/extensions/associationproxy.rst +++ b/doc/build/orm/extensions/associationproxy.rst @@ -510,4 +510,4 @@ API Documentation :members: :undoc-members: -.. autodata:: ASSOCIATION_PROXY \ No newline at end of file +.. autodata:: ASSOCIATION_PROXY diff --git a/doc/build/orm/extensions/mutable.rst b/doc/build/orm/extensions/mutable.rst index 14875cd3c..969411481 100644 --- a/doc/build/orm/extensions/mutable.rst +++ b/doc/build/orm/extensions/mutable.rst @@ -21,7 +21,7 @@ API Reference .. autoclass:: MutableDict :members: - + :undoc-members: diff --git a/doc/build/orm/internals.rst b/doc/build/orm/internals.rst index 78ec2fa8e..4b6802394 100644 --- a/doc/build/orm/internals.rst +++ b/doc/build/orm/internals.rst @@ -11,6 +11,9 @@ sections, are listed here. .. autoclass:: sqlalchemy.orm.state.AttributeState :members: +.. autoclass:: sqlalchemy.orm.util.CascadeOptions + :members: + .. autoclass:: sqlalchemy.orm.instrumentation.ClassManager :members: :inherited-members: @@ -19,6 +22,9 @@ sections, are listed here. :members: :inherited-members: +.. autoclass:: sqlalchemy.orm.properties.ComparableProperty + :members: + .. autoclass:: sqlalchemy.orm.descriptor_props.CompositeProperty :members: diff --git a/doc/build/orm/join_conditions.rst b/doc/build/orm/join_conditions.rst index 5e2c11d1d..c39b7312e 100644 --- a/doc/build/orm/join_conditions.rst +++ b/doc/build/orm/join_conditions.rst @@ -462,7 +462,7 @@ we seek for a load of ``Element.descendants`` to look like:: .. versionadded:: 0.9.5 Support has been added to allow a single-column comparison to itself within a primaryjoin condition, as well as for - primaryjoin conditions that use :meth:`.Operators.like` as the comparison + primaryjoin conditions that use :meth:`.ColumnOperators.like` as the comparison operator. .. _self_referential_many_to_many: diff --git a/doc/build/orm/mapper_config.rst b/doc/build/orm/mapper_config.rst index 671abdfd6..9d584cbab 100644 --- a/doc/build/orm/mapper_config.rst +++ b/doc/build/orm/mapper_config.rst @@ -18,4 +18,4 @@ know how to construct and use rudimentary mappers and relationships. nonstandard_mappings classical versioning - mapping_api \ No newline at end of file + mapping_api diff --git a/doc/build/orm/session_api.rst b/doc/build/orm/session_api.rst index 64ac8c086..3754ac80b 100644 --- a/doc/build/orm/session_api.rst +++ b/doc/build/orm/session_api.rst @@ -1,3 +1,5 @@ +.. module:: sqlalchemy.orm.session + Session API ============ diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 034ee9076..d1fcbef3e 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -299,7 +299,7 @@ not re-compute the column on demand. In order to provide for this explicit query planning, or to use different search strategies, the ``match`` method accepts a ``postgresql_regconfig`` -keyword argument. +keyword argument:: select([mytable.c.id]).where( mytable.c.title.match('somestring', postgresql_regconfig='english') @@ -311,7 +311,7 @@ Emits the equivalent of:: WHERE mytable.title @@ to_tsquery('english', 'somestring') One can also specifically pass in a `'regconfig'` value to the -``to_tsvector()`` command as the initial argument. +``to_tsvector()`` command as the initial argument:: select([mytable.c.id]).where( func.to_tsvector('english', mytable.c.title )\ diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index f67b2e3b0..fe27da8b6 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -66,12 +66,13 @@ in ``/tmp``, or whatever socket directory was specified when PostgreSQL was built. This value can be overridden by passing a pathname to psycopg2, using ``host`` as an additional keyword argument:: - create_engine("postgresql+psycopg2://user:password@/dbname?host=/var/lib/postgresql") + create_engine("postgresql+psycopg2://user:password@/dbname?\ +host=/var/lib/postgresql") See also: -`PQconnectdbParams `_ +`PQconnectdbParams `_ Per-Statement/Connection Execution Options ------------------------------------------- @@ -237,7 +238,7 @@ The psycopg2 dialect supports these constants for isolation level: * ``AUTOCOMMIT`` .. versionadded:: 0.8.2 support for AUTOCOMMIT isolation level when using - psycopg2. + psycopg2. .. seealso:: diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 507e99b2e..0e272dc95 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -596,8 +596,8 @@ class Session(_SessionClassMethods): .. versionadded:: 0.9.0 :param query_cls: Class which should be used to create new Query - objects, as returned by the :meth:`~.Session.query` method. - Defaults to :class:`.Query`. + objects, as returned by the :meth:`~.Session.query` method. + Defaults to :class:`.Query`. :param twophase: When ``True``, all transactions will be started as a "two phase" transaction, i.e. using the "two phase" semantics diff --git a/lib/sqlalchemy/pool.py b/lib/sqlalchemy/pool.py index a174df784..a147685d9 100644 --- a/lib/sqlalchemy/pool.py +++ b/lib/sqlalchemy/pool.py @@ -917,9 +917,9 @@ class QueuePool(Pool): on returning a connection. Defaults to 30. :param \**kw: Other keyword arguments including - :paramref:`.Pool.recycle`, :paramref:`.Pool.echo`, - :paramref:`.Pool.reset_on_return` and others are passed to the - :class:`.Pool` constructor. + :paramref:`.Pool.recycle`, :paramref:`.Pool.echo`, + :paramref:`.Pool.reset_on_return` and others are passed to the + :class:`.Pool` constructor. """ Pool.__init__(self, creator, **kw) diff --git a/lib/sqlalchemy/sql/expression.py b/lib/sqlalchemy/sql/expression.py index 2ffc5468c..2218bd660 100644 --- a/lib/sqlalchemy/sql/expression.py +++ b/lib/sqlalchemy/sql/expression.py @@ -47,7 +47,7 @@ from .base import ColumnCollection, Generative, Executable, \ from .selectable import Alias, Join, Select, Selectable, TableClause, \ CompoundSelect, CTE, FromClause, FromGrouping, SelectBase, \ alias, GenerativeSelect, \ - subquery, HasPrefixes, Exists, ScalarSelect, TextAsFrom + subquery, HasPrefixes, HasSuffixes, Exists, ScalarSelect, TextAsFrom from .dml import Insert, Update, Delete, UpdateBase, ValuesBase diff --git a/lib/sqlalchemy/sql/schema.py b/lib/sqlalchemy/sql/schema.py index b90f7fc53..b134b3053 100644 --- a/lib/sqlalchemy/sql/schema.py +++ b/lib/sqlalchemy/sql/schema.py @@ -2345,6 +2345,14 @@ def _to_schema_column_or_string(element): class ColumnCollectionMixin(object): + columns = None + """A :class:`.ColumnCollection` of :class:`.Column` objects. + + This collection represents the columns which are referred to by + this object. + + """ + def __init__(self, *columns): self.columns = ColumnCollection() self._pending_colargs = [_to_schema_column_or_string(c) diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 9a2de39b4..7bb5c5515 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -1657,7 +1657,7 @@ class NullType(TypeEngine): class MatchType(Boolean): """Refers to the return type of the MATCH operator. - As the :meth:`.Operators.match` is probably the most open-ended + As the :meth:`.ColumnOperators.match` is probably the most open-ended operator in generic SQLAlchemy Core, we can't assume the return type at SQL evaluation time, as MySQL returns a floating point, not a boolean, and other backends might do something different. So this type -- cgit v1.2.1 From 5343d24fee5219de002a8efba8bc882f8b3d4b5b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 16:54:32 -0500 Subject: corrections --- doc/build/changelog/changelog_10.rst | 2 +- doc/build/conf.py | 8 +++++--- doc/build/core/connections.rst | 9 +++++--- doc/build/core/constraints.rst | 4 ++-- doc/build/core/ddl.rst | 33 ++++++++++++++++++------------ doc/build/core/functions.rst | 3 ++- doc/build/core/internals.rst | 7 +++++++ doc/build/orm/internals.rst | 3 +++ lib/sqlalchemy/dialects/postgresql/base.py | 2 +- lib/sqlalchemy/engine/__init__.py | 1 + lib/sqlalchemy/engine/base.py | 2 +- lib/sqlalchemy/ext/hybrid.py | 2 +- lib/sqlalchemy/schema.py | 1 + lib/sqlalchemy/sql/ddl.py | 2 +- lib/sqlalchemy/sql/type_api.py | 2 +- 15 files changed, 53 insertions(+), 28 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index ceed4d912..3564ecde1 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -861,7 +861,7 @@ .. change:: :tags: bug, orm, py3k - The :class:`.IdentityMap` exposed from :class:`.Session.identity_map` + The :class:`.IdentityMap` exposed from :attr:`.Session.identity_map` now returns lists for ``items()`` and ``values()`` in Py3K. Early porting to Py3K here had these returning iterators, when they technically should be "iterable views"..for now, lists are OK. diff --git a/doc/build/conf.py b/doc/build/conf.py index 02784bdae..22b377fa1 100644 --- a/doc/build/conf.py +++ b/doc/build/conf.py @@ -37,7 +37,7 @@ extensions = [ 'zzzeeksphinx', 'changelog', 'sphinx_paramlinks', - 'corrections' + #'corrections' ] # Add any paths that contain templates here, relative to this directory. @@ -79,7 +79,9 @@ autodocmods_convert_modname = { "sqlalchemy.sql.selectable": "sqlalchemy.sql.expression", "sqlalchemy.sql.dml": "sqlalchemy.sql.expression", "sqlalchemy.sql.ddl": "sqlalchemy.schema", - "sqlalchemy.sql.base": "sqlalchemy.sql.expression" + "sqlalchemy.sql.base": "sqlalchemy.sql.expression", + "sqlalchemy.engine.base": "sqlalchemy.engine", + "sqlalchemy.engine.result": "sqlalchemy.engine", } autodocmods_convert_modname_w_class = { @@ -192,7 +194,7 @@ html_title = "%s %s Documentation" % (project, version) # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['static'] +html_static_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/build/core/connections.rst b/doc/build/core/connections.rst index 248309a2e..bb04b9496 100644 --- a/doc/build/core/connections.rst +++ b/doc/build/core/connections.rst @@ -559,6 +559,9 @@ The above will respond to ``create_engine("mysql+foodialect://")`` and load the Connection / Engine API ======================= +.. autoclass:: BaseRowProxy + :members: + .. autoclass:: Connection :members: @@ -568,16 +571,16 @@ Connection / Engine API .. autoclass:: Engine :members: -.. autoclass:: sqlalchemy.engine.ExceptionContext +.. autoclass:: ExceptionContext :members: .. autoclass:: NestedTransaction :members: -.. autoclass:: sqlalchemy.engine.ResultProxy +.. autoclass:: ResultProxy :members: -.. autoclass:: sqlalchemy.engine.RowProxy +.. autoclass:: RowProxy :members: .. autoclass:: Transaction diff --git a/doc/build/core/constraints.rst b/doc/build/core/constraints.rst index 554d003bb..15d0405fe 100644 --- a/doc/build/core/constraints.rst +++ b/doc/build/core/constraints.rst @@ -7,11 +7,11 @@ Defining Constraints and Indexes ================================= -.. _metadata_foreignkeys: - This section will discuss SQL :term:`constraints` and indexes. In SQLAlchemy the key classes include :class:`.ForeignKeyConstraint` and :class:`.Index`. +.. _metadata_foreignkeys: + Defining Foreign Keys --------------------- diff --git a/doc/build/core/ddl.rst b/doc/build/core/ddl.rst index cee6f876e..6607ac9f8 100644 --- a/doc/build/core/ddl.rst +++ b/doc/build/core/ddl.rst @@ -223,65 +223,72 @@ DDL Expression Constructs API .. autoclass:: DDLElement :members: :undoc-members: - + .. autoclass:: DDL :members: :undoc-members: - + + +.. autoclass:: _DDLCompiles + :members: + + +.. autoclass:: _CreateDropBase + :members: .. autoclass:: CreateTable :members: :undoc-members: - + .. autoclass:: DropTable :members: :undoc-members: - + .. autoclass:: CreateColumn :members: :undoc-members: - + .. autoclass:: CreateSequence :members: :undoc-members: - + .. autoclass:: DropSequence :members: :undoc-members: - + .. autoclass:: CreateIndex :members: :undoc-members: - + .. autoclass:: DropIndex :members: :undoc-members: - + .. autoclass:: AddConstraint :members: :undoc-members: - + .. autoclass:: DropConstraint :members: :undoc-members: - + .. autoclass:: CreateSchema :members: :undoc-members: - + .. autoclass:: DropSchema :members: :undoc-members: - + diff --git a/doc/build/core/functions.rst b/doc/build/core/functions.rst index d284d125f..90164850d 100644 --- a/doc/build/core/functions.rst +++ b/doc/build/core/functions.rst @@ -22,6 +22,7 @@ return types are in use. .. automodule:: sqlalchemy.sql.functions :members: :undoc-members: - + :exclude-members: func + diff --git a/doc/build/core/internals.rst b/doc/build/core/internals.rst index 1a85e9e6c..81b4f1a81 100644 --- a/doc/build/core/internals.rst +++ b/doc/build/core/internals.rst @@ -7,6 +7,9 @@ Some key internal constructs are listed here. .. currentmodule: sqlalchemy +.. autoclass:: sqlalchemy.schema.ColumnCollectionMixin + :members: + .. autoclass:: sqlalchemy.engine.interfaces.Compiled :members: @@ -29,6 +32,10 @@ Some key internal constructs are listed here. :members: +.. autoclass:: sqlalchemy.log.Identified + :members: + + .. autoclass:: sqlalchemy.sql.compiler.IdentifierPreparer :members: diff --git a/doc/build/orm/internals.rst b/doc/build/orm/internals.rst index 4b6802394..07cc2b472 100644 --- a/doc/build/orm/internals.rst +++ b/doc/build/orm/internals.rst @@ -33,6 +33,9 @@ sections, are listed here. :members: +.. autoclass:: sqlalchemy.orm.identity.IdentityMap + :members: + .. autoclass:: sqlalchemy.orm.base.InspectionAttr :members: diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index d1fcbef3e..d870dd295 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -266,7 +266,7 @@ will emit to the database:: The Postgresql text search functions such as ``to_tsquery()`` and ``to_tsvector()`` are available -explicitly using the standard :attr:`.func` construct. For example:: +explicitly using the standard :data:`.func` construct. For example:: select([ func.to_tsvector('fat cats ate rats').match('cat & rat') diff --git a/lib/sqlalchemy/engine/__init__.py b/lib/sqlalchemy/engine/__init__.py index cf75871bf..3857bdf1e 100644 --- a/lib/sqlalchemy/engine/__init__.py +++ b/lib/sqlalchemy/engine/__init__.py @@ -72,6 +72,7 @@ from .base import ( ) from .result import ( + BaseRowProxy, BufferedColumnResultProxy, BufferedColumnRow, BufferedRowResultProxy, diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 918ee0e37..ee8267c5c 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -743,7 +743,7 @@ class Connection(Connectable): a subclass of :class:`.Executable`, such as a :func:`~.expression.select` construct * a :class:`.FunctionElement`, such as that generated - by :attr:`.func`, will be automatically wrapped in + by :data:`.func`, will be automatically wrapped in a SELECT statement, which is then executed. * a :class:`.DDLElement` object * a :class:`.DefaultGenerator` object diff --git a/lib/sqlalchemy/ext/hybrid.py b/lib/sqlalchemy/ext/hybrid.py index e2739d1de..d89a13fc9 100644 --- a/lib/sqlalchemy/ext/hybrid.py +++ b/lib/sqlalchemy/ext/hybrid.py @@ -145,7 +145,7 @@ usage of the absolute value function:: return func.abs(cls.length) / 2 Above the Python function ``abs()`` is used for instance-level -operations, the SQL function ``ABS()`` is used via the :attr:`.func` +operations, the SQL function ``ABS()`` is used via the :data:`.func` object for class-level expressions:: >>> i1.radius diff --git a/lib/sqlalchemy/schema.py b/lib/sqlalchemy/schema.py index 4b6ad1988..95ebd05db 100644 --- a/lib/sqlalchemy/schema.py +++ b/lib/sqlalchemy/schema.py @@ -35,6 +35,7 @@ from .sql.schema import ( UniqueConstraint, _get_table_key, ColumnCollectionConstraint, + ColumnCollectionMixin ) diff --git a/lib/sqlalchemy/sql/ddl.py b/lib/sqlalchemy/sql/ddl.py index 1f2c448ea..534322c8d 100644 --- a/lib/sqlalchemy/sql/ddl.py +++ b/lib/sqlalchemy/sql/ddl.py @@ -370,7 +370,7 @@ class DDL(DDLElement): :class:`.DDLEvents` - :mod:`sqlalchemy.event` + :ref:`event_toplevel` """ diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py index d414daf2a..03fed3878 100644 --- a/lib/sqlalchemy/sql/type_api.py +++ b/lib/sqlalchemy/sql/type_api.py @@ -252,7 +252,7 @@ class TypeEngine(Visitable): The construction of :meth:`.TypeEngine.with_variant` is always from the "fallback" type to that which is dialect specific. The returned type is an instance of :class:`.Variant`, which - itself provides a :meth:`~sqlalchemy.types.Variant.with_variant` + itself provides a :meth:`.Variant.with_variant` that can be called repeatedly. :param type_: a :class:`.TypeEngine` that will be selected -- cgit v1.2.1 From 3ccae267894804b104171f2fee9355cd292e0dbf Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 17:17:05 -0500 Subject: remove pipe... --- doc/build/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/index.rst b/doc/build/index.rst index 8b60ef9b9..55dba45fe 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -31,7 +31,7 @@ of Python objects, proceed first to the tutorial. * **ORM Configuration:** :doc:`Mapper Configuration ` | - :doc:`Relationship Configuration ` | + :doc:`Relationship Configuration ` * **Configuration Extensions:** :doc:`Declarative Extension ` | -- cgit v1.2.1 From 31e2fe75d947478209556562be0b79473a288430 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 17:45:58 -0500 Subject: - remove private superclasses from docs in favor of fixing zzzeeksphinx to omit these from warning Conflicts: doc/build/orm/internals.rst --- doc/build/core/connections.rst | 3 --- doc/build/core/constraints.rst | 4 ---- doc/build/core/ddl.rst | 7 ------- doc/build/orm/internals.rst | 1 - doc/build/requirements.txt | 2 +- 5 files changed, 1 insertion(+), 16 deletions(-) diff --git a/doc/build/core/connections.rst b/doc/build/core/connections.rst index bb04b9496..5f0bb4972 100644 --- a/doc/build/core/connections.rst +++ b/doc/build/core/connections.rst @@ -559,9 +559,6 @@ The above will respond to ``create_engine("mysql+foodialect://")`` and load the Connection / Engine API ======================= -.. autoclass:: BaseRowProxy - :members: - .. autoclass:: Connection :members: diff --git a/doc/build/core/constraints.rst b/doc/build/core/constraints.rst index 15d0405fe..9bf510d6a 100644 --- a/doc/build/core/constraints.rst +++ b/doc/build/core/constraints.rst @@ -439,14 +439,10 @@ Constraints API :members: :inherited-members: -.. autoclass:: ColumnCollectionConstraint - :members: - .. autoclass:: ForeignKey :members: :inherited-members: - .. autoclass:: ForeignKeyConstraint :members: :inherited-members: diff --git a/doc/build/core/ddl.rst b/doc/build/core/ddl.rst index 6607ac9f8..b8bdd1a20 100644 --- a/doc/build/core/ddl.rst +++ b/doc/build/core/ddl.rst @@ -230,13 +230,6 @@ DDL Expression Constructs API :undoc-members: -.. autoclass:: _DDLCompiles - :members: - - -.. autoclass:: _CreateDropBase - :members: - .. autoclass:: CreateTable :members: :undoc-members: diff --git a/doc/build/orm/internals.rst b/doc/build/orm/internals.rst index 07cc2b472..bead784a3 100644 --- a/doc/build/orm/internals.rst +++ b/doc/build/orm/internals.rst @@ -32,7 +32,6 @@ sections, are listed here. .. autoclass:: sqlalchemy.orm.attributes.Event :members: - .. autoclass:: sqlalchemy.orm.identity.IdentityMap :members: diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index 3c26bea70..da9a1bcc9 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ changelog>=0.3.4 sphinx-paramlinks>=0.2.2 -zzzeeksphinx>=1.0.1 +zzzeeksphinx>=1.0.3 -- cgit v1.2.1 From d11c7a197e556eda873b267d20a03e317477e510 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 17:56:40 -0500 Subject: - another bump --- doc/build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index da9a1bcc9..bdbd832e4 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ changelog>=0.3.4 sphinx-paramlinks>=0.2.2 -zzzeeksphinx>=1.0.3 +zzzeeksphinx>=1.0.4 -- cgit v1.2.1 From 0bd632804eae635d793175a959294f49f3538806 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 27 Dec 2014 19:05:08 -0500 Subject: - fix links for loading, add a redirect page bump foo --- doc/build/glossary.rst | 4 ++-- doc/build/orm/loading.rst | 3 +++ doc/build/requirements.txt | 2 +- lib/sqlalchemy/orm/query.py | 4 ++-- lib/sqlalchemy/orm/relationships.py | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 doc/build/orm/loading.rst diff --git a/doc/build/glossary.rst b/doc/build/glossary.rst index ab9e92d26..2e5dc67f3 100644 --- a/doc/build/glossary.rst +++ b/doc/build/glossary.rst @@ -146,7 +146,7 @@ Glossary :term:`N plus one problem` - :doc:`orm/loading` + :doc:`orm/loading_relationships` mapping mapped @@ -175,7 +175,7 @@ Glossary .. seealso:: - :doc:`orm/loading` + :doc:`orm/loading_relationships` polymorphic polymorphically diff --git a/doc/build/orm/loading.rst b/doc/build/orm/loading.rst new file mode 100644 index 000000000..0aca6cd0c --- /dev/null +++ b/doc/build/orm/loading.rst @@ -0,0 +1,3 @@ +:orphan: + +Moved! :doc:`/orm/loading_relationships` \ No newline at end of file diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index bdbd832e4..c84d342d6 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ changelog>=0.3.4 sphinx-paramlinks>=0.2.2 -zzzeeksphinx>=1.0.4 +zzzeeksphinx>=1.0.5 diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 6cade322a..d2bdf5a00 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -811,7 +811,7 @@ class Query(object): foreign-key-to-primary-key criterion, will also use an operation equivalent to :meth:`~.Query.get` in order to retrieve the target value from the local identity map - before querying the database. See :doc:`/orm/loading` + before querying the database. See :doc:`/orm/loading_relationships` for further details on relationship loading. :param ident: A scalar or tuple value representing @@ -1100,7 +1100,7 @@ class Query(object): Most supplied options regard changing how column- and relationship-mapped attributes are loaded. See the sections - :ref:`deferred` and :doc:`/orm/loading` for reference + :ref:`deferred` and :doc:`/orm/loading_relationships` for reference documentation. """ diff --git a/lib/sqlalchemy/orm/relationships.py b/lib/sqlalchemy/orm/relationships.py index 86f1b3f82..d3ae107b9 100644 --- a/lib/sqlalchemy/orm/relationships.py +++ b/lib/sqlalchemy/orm/relationships.py @@ -528,7 +528,7 @@ class RelationshipProperty(StrategizedProperty): .. seealso:: - :doc:`/orm/loading` - Full documentation on relationship loader + :doc:`/orm/loading_relationships` - Full documentation on relationship loader configuration. :ref:`dynamic_relationship` - detail on the ``dynamic`` option. -- cgit v1.2.1 From 60368fdd9f278492099f7c8108456ecbb44c76ef Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 28 Dec 2014 10:43:39 -0500 Subject: bump --- doc/build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index c84d342d6..c782871f5 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ changelog>=0.3.4 sphinx-paramlinks>=0.2.2 -zzzeeksphinx>=1.0.5 +zzzeeksphinx>=1.0.6 -- cgit v1.2.1 From 1cb24b37421d413045daccfc819d9fa0e61dd4c4 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 28 Dec 2014 11:06:47 -0500 Subject: bump --- doc/build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index c782871f5..798ee9300 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ changelog>=0.3.4 sphinx-paramlinks>=0.2.2 -zzzeeksphinx>=1.0.6 +zzzeeksphinx>=1.0.7 -- cgit v1.2.1 From 3a620a304ef4b323871b21c83776078b9aae8135 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 28 Dec 2014 17:11:36 -0500 Subject: - see if we can get RTD to use this for now --- doc/build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index 798ee9300..3f87e68ea 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -1,3 +1,3 @@ changelog>=0.3.4 sphinx-paramlinks>=0.2.2 -zzzeeksphinx>=1.0.7 +git+https://bitbucket.org/zzzeek/zzzeeksphinx.git -- cgit v1.2.1 From 87a1af4efe5bb515d9e687e2f7dfc84dfb8ee522 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 29 Dec 2014 20:01:21 -0500 Subject: - fix some RST whitespace syntactical issues in toctrees - have the topmost toctree only include page names - add glossary to toctree, remove search by itself --- doc/build/changelog/index.rst | 8 ++++---- doc/build/contents.rst | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/build/changelog/index.rst b/doc/build/changelog/index.rst index 0f5d090a3..8c5be99b8 100644 --- a/doc/build/changelog/index.rst +++ b/doc/build/changelog/index.rst @@ -10,15 +10,15 @@ Current Migration Guide ------------------------ .. toctree:: - :maxdepth: 1 + :titlesonly: - migration_10 + migration_10 Change logs ----------- .. toctree:: - :maxdepth: 2 + :titlesonly: changelog_10 changelog_09 @@ -36,7 +36,7 @@ Older Migration Guides ---------------------- .. toctree:: - :maxdepth: 1 + :titlesonly: migration_09 migration_08 diff --git a/doc/build/contents.rst b/doc/build/contents.rst index 95b5e9a19..a7277cf90 100644 --- a/doc/build/contents.rst +++ b/doc/build/contents.rst @@ -7,17 +7,18 @@ Full table of contents. For a high level overview of all documentation, see :ref:`index_toplevel`. .. toctree:: - :maxdepth: 3 + :titlesonly: + :includehidden: - intro - orm/index - core/index - dialects/index - faq/index - changelog/index + intro + orm/index + core/index + dialects/index + faq/index + changelog/index Indices and tables ------------------ +* :ref:`glossary` * :ref:`genindex` -* :ref:`search` -- cgit v1.2.1 From da1aa2590851bd5ddc58218fab0e8234d16db97c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 1 Jan 2015 13:14:05 -0500 Subject: - remove the "edges" from the message here. It's illegible --- lib/sqlalchemy/exc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlalchemy/exc.py b/lib/sqlalchemy/exc.py index 3271d09d4..d6355a212 100644 --- a/lib/sqlalchemy/exc.py +++ b/lib/sqlalchemy/exc.py @@ -63,7 +63,7 @@ class CircularDependencyError(SQLAlchemyError): """ def __init__(self, message, cycles, edges, msg=None): if msg is None: - message += " Cycles: %r all edges: %r" % (cycles, edges) + message += " (%s)" % ", ".join(repr(s) for s in cycles) else: message = msg SQLAlchemyError.__init__(self, message) -- cgit v1.2.1 From 8f5e4acbf693a375ad687977188a32bc941fd33b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 1 Jan 2015 13:24:32 -0500 Subject: - Added a new accessor :attr:`.Table.foreign_key_constraints` to complement the :attr:`.Table.foreign_keys` collection, as well as :attr:`.ForeignKeyConstraint.referred_table`. --- doc/build/changelog/changelog_10.rst | 7 +++++++ lib/sqlalchemy/sql/schema.py | 27 ++++++++++++++++++++++++ test/sql/test_metadata.py | 40 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 3564ecde1..4b3a17367 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,13 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, schema + + Added a new accessor :attr:`.Table.foreign_key_constraints` + to complement the :attr:`.Table.foreign_keys` collection, + as well as :attr:`.ForeignKeyConstraint.referred_table`. + .. change:: :tags: bug, sqlite :tickets: 3244, 3261 diff --git a/lib/sqlalchemy/sql/schema.py b/lib/sqlalchemy/sql/schema.py index b134b3053..71a0c2780 100644 --- a/lib/sqlalchemy/sql/schema.py +++ b/lib/sqlalchemy/sql/schema.py @@ -516,6 +516,19 @@ class Table(DialectKWArgs, SchemaItem, TableClause): """ return sorted(self.constraints, key=lambda c: c._creation_order) + @property + def foreign_key_constraints(self): + """:class:`.ForeignKeyConstraint` objects referred to by this + :class:`.Table`. + + This list is produced from the collection of :class:`.ForeignKey` + objects currently associated. + + .. versionadded:: 1.0.0 + + """ + return set(fkc.constraint for fkc in self.foreign_keys) + def _init_existing(self, *args, **kwargs): autoload_with = kwargs.pop('autoload_with', None) autoload = kwargs.pop('autoload', autoload_with is not None) @@ -2632,6 +2645,20 @@ class ForeignKeyConstraint(ColumnCollectionConstraint): else: return None + @property + def referred_table(self): + """The :class:`.Table` object to which this + :class:`.ForeignKeyConstraint references. + + This is a dynamically calculated attribute which may not be available + if the constraint and/or parent table is not yet associated with + a metadata collection that contains the referred table. + + .. versionadded:: 1.0.0 + + """ + return self.elements[0].column.table + def _validate_dest_table(self, table): table_keys = set([elem._table_key() for elem in self.elements]) diff --git a/test/sql/test_metadata.py b/test/sql/test_metadata.py index 52ecf88c5..cc7d0eb4f 100644 --- a/test/sql/test_metadata.py +++ b/test/sql/test_metadata.py @@ -1196,6 +1196,30 @@ class TableTest(fixtures.TestBase, AssertsCompiledSQL): t.info['bar'] = 'zip' assert t.info['bar'] == 'zip' + def test_foreign_key_constraints_collection(self): + metadata = MetaData() + t1 = Table('foo', metadata, Column('a', Integer)) + eq_(t1.foreign_key_constraints, set()) + + fk1 = ForeignKey('q.id') + fk2 = ForeignKey('j.id') + fk3 = ForeignKeyConstraint(['b', 'c'], ['r.x', 'r.y']) + + t1.append_column(Column('b', Integer, fk1)) + eq_( + t1.foreign_key_constraints, + set([fk1.constraint])) + + t1.append_column(Column('c', Integer, fk2)) + eq_( + t1.foreign_key_constraints, + set([fk1.constraint, fk2.constraint])) + + t1.append_constraint(fk3) + eq_( + t1.foreign_key_constraints, + set([fk1.constraint, fk2.constraint, fk3])) + def test_c_immutable(self): m = MetaData() t1 = Table('t', m, Column('x', Integer), Column('y', Integer)) @@ -1947,6 +1971,22 @@ class ConstraintTest(fixtures.TestBase): assert s1.c.a.references(t1.c.a) assert not s1.c.a.references(t1.c.b) + def test_referred_table_accessor(self): + t1, t2, t3 = self._single_fixture() + fkc = list(t2.foreign_key_constraints)[0] + is_(fkc.referred_table, t1) + + def test_referred_table_accessor_not_available(self): + t1 = Table('t', MetaData(), Column('x', ForeignKey('q.id'))) + fkc = list(t1.foreign_key_constraints)[0] + assert_raises_message( + exc.InvalidRequestError, + "Foreign key associated with column 't.x' could not find " + "table 'q' with which to generate a foreign key to target " + "column 'id'", + getattr, fkc, "referred_table" + ) + def test_related_column_not_present_atfirst_ok(self): m = MetaData() base_table = Table("base", m, -- cgit v1.2.1 From 21f47124ab433cc74fa0a72efcc8a6c1e9c37db5 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 1 Jan 2015 13:47:08 -0500 Subject: - restate sort_tables in terms of a more fine grained sort_tables_and_constraints function. - The DDL generation system of :meth:`.MetaData.create_all` and :meth:`.Metadata.drop_all` has been enhanced to in most cases automatically handle the case of mutually dependent foreign key constraints; the need for the :paramref:`.ForeignKeyConstraint.use_alter` flag is greatly reduced. The system also works for constraints which aren't given a name up front; only in the case of DROP is a name required for at least one of the constraints involved in the cycle. fixes #3282 --- doc/build/changelog/changelog_10.rst | 17 ++ doc/build/changelog/migration_10.rst | 29 +++ doc/build/core/constraints.rst | 185 ++++++++++++++++--- doc/build/core/ddl.rst | 4 + lib/sqlalchemy/dialects/mysql/base.py | 4 +- lib/sqlalchemy/dialects/sqlite/base.py | 12 ++ lib/sqlalchemy/engine/reflection.py | 69 ++++++- lib/sqlalchemy/schema.py | 4 +- lib/sqlalchemy/sql/compiler.py | 29 ++- lib/sqlalchemy/sql/ddl.py | 264 +++++++++++++++++++++++---- lib/sqlalchemy/sql/schema.py | 62 +++++-- lib/sqlalchemy/testing/__init__.py | 3 +- lib/sqlalchemy/testing/plugin/plugin_base.py | 14 +- lib/sqlalchemy/testing/util.py | 55 ++++++ test/orm/test_cycles.py | 18 +- test/sql/test_constraints.py | 207 ++++++++++++++++++--- test/sql/test_ddlemit.py | 67 ++++++- 17 files changed, 909 insertions(+), 134 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 4b3a17367..95eaff0f1 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,23 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, schema + :tickets: 3282 + + The DDL generation system of :meth:`.MetaData.create_all` + and :meth:`.MetaData.drop_all` has been enhanced to in most + cases automatically handle the case of mutually dependent + foreign key constraints; the need for the + :paramref:`.ForeignKeyConstraint.use_alter` flag is greatly + reduced. The system also works for constraints which aren't given + a name up front; only in the case of DROP is a name required for + at least one of the constraints involved in the cycle. + + .. seealso:: + + :ref:`feature_3282` + .. change:: :tags: feature, schema diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 829d04c51..f9c26017c 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -488,6 +488,35 @@ wishes to support the new feature should now call upon the ``._limit_clause`` and ``._offset_clause`` attributes to receive the full SQL expression, rather than the integer value. +.. _feature_3282: + +The ``use_alter`` flag on ``ForeignKeyConstraint`` is no longer needed +---------------------------------------------------------------------- + +The :meth:`.MetaData.create_all` and :meth:`.MetaData.drop_all` methods will +now make use of a system that automatically renders an ALTER statement +for foreign key constraints that are involved in mutually-dependent cycles +between tables, without the +need to specify :paramref:`.ForeignKeyConstraint.use_alter`. Additionally, +the foreign key constraints no longer need to have a name in order to be +created via ALTER; only the DROP operation requires a name. In the case +of a DROP, the feature will ensure that only constraints which have +explicit names are actually included as ALTER statements. In the +case of an unresolvable cycle within a DROP, the system emits +a succinct and clear error message now if the DROP cannot proceed. + +The :paramref:`.ForeignKeyConstraint.use_alter` and +:paramref:`.ForeignKey.use_alter` flags remain in place, and continue to have +the same effect of establishing those constraints for which ALTER is +required during a CREATE/DROP scenario. + +.. seealso:: + + :ref:`use_alter` - full description of the new behavior. + + +:ticket:`3282` + .. _change_2051: .. _feature_insert_from_select_defaults: diff --git a/doc/build/core/constraints.rst b/doc/build/core/constraints.rst index 9bf510d6a..a11300100 100644 --- a/doc/build/core/constraints.rst +++ b/doc/build/core/constraints.rst @@ -95,40 +95,179 @@ foreign key referencing two columns. Creating/Dropping Foreign Key Constraints via ALTER ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In all the above examples, the :class:`~sqlalchemy.schema.ForeignKey` object -causes the "REFERENCES" keyword to be added inline to a column definition -within a "CREATE TABLE" statement when -:func:`~sqlalchemy.schema.MetaData.create_all` is issued, and -:class:`~sqlalchemy.schema.ForeignKeyConstraint` invokes the "CONSTRAINT" -keyword inline with "CREATE TABLE". There are some cases where this is -undesirable, particularly when two tables reference each other mutually, each -with a foreign key referencing the other. In such a situation at least one of -the foreign key constraints must be generated after both tables have been -built. To support such a scheme, :class:`~sqlalchemy.schema.ForeignKey` and -:class:`~sqlalchemy.schema.ForeignKeyConstraint` offer the flag -``use_alter=True``. When using this flag, the constraint will be generated -using a definition similar to "ALTER TABLE ADD CONSTRAINT -...". Since a name is required, the ``name`` attribute must also be specified. -For example:: - - node = Table('node', meta, +The behavior we've seen in tutorials and elsewhere involving +foreign keys with DDL illustrates that the constraints are typically +rendered "inline" within the CREATE TABLE statement, such as: + +.. sourcecode:: sql + + CREATE TABLE addresses ( + id INTEGER NOT NULL, + user_id INTEGER, + email_address VARCHAR NOT NULL, + PRIMARY KEY (id), + CONSTRAINT user_id_fk FOREIGN KEY(user_id) REFERENCES users (id) + ) + +The ``CONSTRAINT .. FOREIGN KEY`` directive is used to create the constraint +in an "inline" fashion within the CREATE TABLE definition. The +:meth:`.MetaData.create_all` and :meth:`.MetaData.drop_all` methods do +this by default, using a topological sort of all the :class:`.Table` objects +involved such that tables are created and dropped in order of their foreign +key dependency (this sort is also available via the +:attr:`.MetaData.sorted_tables` accessor). + +This approach can't work when two or more foreign key constraints are +involved in a "dependency cycle", where a set of tables +are mutually dependent on each other, assuming the backend enforces foreign +keys (always the case except on SQLite, MySQL/MyISAM). The methods will +therefore break out constraints in such a cycle into separate ALTER +statements, on all backends other than SQLite which does not support +most forms of ALTER. Given a schema like:: + + node = Table( + 'node', metadata, Column('node_id', Integer, primary_key=True), - Column('primary_element', Integer, - ForeignKey('element.element_id', use_alter=True, name='fk_node_element_id') + Column( + 'primary_element', Integer, + ForeignKey('element.element_id') ) ) - element = Table('element', meta, + element = Table( + 'element', metadata, Column('element_id', Integer, primary_key=True), Column('parent_node_id', Integer), ForeignKeyConstraint( - ['parent_node_id'], - ['node.node_id'], - use_alter=True, + ['parent_node_id'], ['node.node_id'], name='fk_element_parent_node_id' ) ) +When we call upon :meth:`.MetaData.create_all` on a backend such as the +Postgresql backend, the cycle between these two tables is resolved and the +constraints are created separately: + +.. sourcecode:: pycon+sql + + >>> with engine.connect() as conn: + ... metadata.create_all(conn, checkfirst=False) + {opensql}CREATE TABLE element ( + element_id SERIAL NOT NULL, + parent_node_id INTEGER, + PRIMARY KEY (element_id) + ) + + CREATE TABLE node ( + node_id SERIAL NOT NULL, + primary_element INTEGER, + PRIMARY KEY (node_id) + ) + + ALTER TABLE element ADD CONSTRAINT fk_element_parent_node_id + FOREIGN KEY(parent_node_id) REFERENCES node (node_id) + ALTER TABLE node ADD FOREIGN KEY(primary_element) + REFERENCES element (element_id) + {stop} + +In order to emit DROP for these tables, the same logic applies, however +note here that in SQL, to emit DROP CONSTRAINT requires that the constraint +has a name. In the case of the ``'node'`` table above, we haven't named +this constraint; the system will therefore attempt to emit DROP for only +those constraints that are named: + +.. NOTE: the parser is doing something wrong with the DROP here, + if the "DROP TABLE element" is second, the "t" is being chopped off; + it is specific to the letter "t". Look into this at some point + +.. sourcecode:: pycon+sql + + >>> with engine.connect() as conn: + ... metadata.drop_all(conn, checkfirst=False) + {opensql}ALTER TABLE element DROP CONSTRAINT fk_element_parent_node_id + DROP TABLE element + DROP TABLE node + {stop} + + +In the case where the cycle cannot be resolved, such as if we hadn't applied +a name to either constraint here, we will receive the following error:: + + sqlalchemy.exc.CircularDependencyError: Can't sort tables for DROP; + an unresolvable foreign key dependency exists between tables: + element, node. Please ensure that the ForeignKey and ForeignKeyConstraint + objects involved in the cycle have names so that they can be dropped + using DROP CONSTRAINT. + +This error only applies to the DROP case as we can emit "ADD CONSTRAINT" +in the CREATE case without a name; the database typically assigns one +automatically. + +The :paramref:`.ForeignKeyConstraint.use_alter` and +:paramref:`.ForeignKey.use_alter` keyword arguments can be used +to manually resolve dependency cycles. We can add this flag only to +the ``'element'`` table as follows:: + + element = Table( + 'element', metadata, + Column('element_id', Integer, primary_key=True), + Column('parent_node_id', Integer), + ForeignKeyConstraint( + ['parent_node_id'], ['node.node_id'], + use_alter=True, name='fk_element_parent_node_id' + ) + ) + +in our CREATE DDL we will see the ALTER statement only for this constraint, +and not the other one: + +.. sourcecode:: pycon+sql + + >>> with engine.connect() as conn: + ... metadata.create_all(conn, checkfirst=False) + {opensql}CREATE TABLE element ( + element_id SERIAL NOT NULL, + parent_node_id INTEGER, + PRIMARY KEY (element_id) + ) + + CREATE TABLE node ( + node_id SERIAL NOT NULL, + primary_element INTEGER, + PRIMARY KEY (node_id), + FOREIGN KEY(primary_element) REFERENCES element (element_id) + ) + + ALTER TABLE element ADD CONSTRAINT fk_element_parent_node_id + FOREIGN KEY(parent_node_id) REFERENCES node (node_id) + {stop} + +:paramref:`.ForeignKeyConstraint.use_alter` and +:paramref:`.ForeignKey.use_alter`, when used in conjunction with a drop +operation, will require that the constraint is named, else an error +like the following is generated:: + + sqlalchemy.exc.CompileError: Can't emit DROP CONSTRAINT for constraint + ForeignKeyConstraint(...); it has no name + +.. versionchanged:: 1.0.0 - The DDL system invoked by + :meth:`.MetaData.create_all` + and :meth:`.MetaData.drop_all` will now automatically resolve mutually + depdendent foreign keys between tables declared by + :class:`.ForeignKeyConstraint` and :class:`.ForeignKey` objects, without + the need to explicitly set the :paramref:`.ForeignKeyConstraint.use_alter` + flag. + +.. versionchanged:: 1.0.0 - The :paramref:`.ForeignKeyConstraint.use_alter` + flag can be used with an un-named constraint; only the DROP operation + will emit a specific error when actually called upon. + +.. seealso:: + + :ref:`constraint_naming_conventions` + + :func:`.sort_tables_and_constraints` + .. _on_update_on_delete: ON UPDATE and ON DELETE diff --git a/doc/build/core/ddl.rst b/doc/build/core/ddl.rst index b8bdd1a20..0ba2f2806 100644 --- a/doc/build/core/ddl.rst +++ b/doc/build/core/ddl.rst @@ -220,6 +220,10 @@ details. DDL Expression Constructs API ----------------------------- +.. autofunction:: sort_tables + +.. autofunction:: sort_tables_and_constraints + .. autoclass:: DDLElement :members: :undoc-members: diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index c868f58b2..5f990ea4e 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -1767,10 +1767,10 @@ class MySQLCompiler(compiler.SQLCompiler): # creation of foreign key constraints fails." class MySQLDDLCompiler(compiler.DDLCompiler): - def create_table_constraints(self, table): + def create_table_constraints(self, table, **kw): """Get table constraints.""" constraint_string = super( - MySQLDDLCompiler, self).create_table_constraints(table) + MySQLDDLCompiler, self).create_table_constraints(table, **kw) # why self.dialect.name and not 'mysql'? because of drizzle is_innodb = 'engine' in table.dialect_options[self.dialect.name] and \ diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index e0b2875e8..3d7b0788b 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -201,6 +201,15 @@ new connections through the usage of events:: cursor.execute("PRAGMA foreign_keys=ON") cursor.close() +.. warning:: + + When SQLite foreign keys are enabled, it is **not possible** + to emit CREATE or DROP statements for tables that contain + mutually-dependent foreign key constraints; + to emit the DDL for these tables requires that ALTER TABLE be used to + create or drop these constraints separately, for which SQLite has + no support. + .. seealso:: `SQLite Foreign Key Support `_ @@ -208,6 +217,9 @@ new connections through the usage of events:: :ref:`event_toplevel` - SQLAlchemy event API. + :ref:`use_alter` - more information on SQLAlchemy's facilities for handling + mutually-dependent foreign key constraints. + .. _sqlite_type_reflection: Type Reflection diff --git a/lib/sqlalchemy/engine/reflection.py b/lib/sqlalchemy/engine/reflection.py index 25f084c15..6e102aad6 100644 --- a/lib/sqlalchemy/engine/reflection.py +++ b/lib/sqlalchemy/engine/reflection.py @@ -173,7 +173,14 @@ class Inspector(object): passed as ``None``. For special quoting, use :class:`.quoted_name`. :param order_by: Optional, may be the string "foreign_key" to sort - the result on foreign key dependencies. + the result on foreign key dependencies. Does not automatically + resolve cycles, and will raise :class:`.CircularDependencyError` + if cycles exist. + + .. deprecated:: 1.0.0 - see + :meth:`.Inspector.get_sorted_table_and_fkc_names` for a version + of this which resolves foreign key cycles between tables + automatically. .. versionchanged:: 0.8 the "foreign_key" sorting sorts tables in order of dependee to dependent; that is, in creation @@ -183,6 +190,8 @@ class Inspector(object): .. seealso:: + :meth:`.Inspector.get_sorted_table_and_fkc_names` + :attr:`.MetaData.sorted_tables` """ @@ -201,6 +210,64 @@ class Inspector(object): tnames = list(topological.sort(tuples, tnames)) return tnames + def get_sorted_table_and_fkc_names(self, schema=None): + """Return dependency-sorted table and foreign key constraint names in + referred to within a particular schema. + + This will yield 2-tuples of + ``(tablename, [(tname, fkname), (tname, fkname), ...])`` + consisting of table names in CREATE order grouped with the foreign key + constraint names that are not detected as belonging to a cycle. + The final element + will be ``(None, [(tname, fkname), (tname, fkname), ..])`` + which will consist of remaining + foreign key constraint names that would require a separate CREATE + step after-the-fact, based on dependencies between tables. + + .. versionadded:: 1.0.- + + .. seealso:: + + :meth:`.Inspector.get_table_names` + + :func:`.sort_tables_and_constraints` - similar method which works + with an already-given :class:`.MetaData`. + + """ + if hasattr(self.dialect, 'get_table_names'): + tnames = self.dialect.get_table_names( + self.bind, schema, info_cache=self.info_cache) + else: + tnames = self.engine.table_names(schema) + + tuples = set() + remaining_fkcs = set() + + fknames_for_table = {} + for tname in tnames: + fkeys = self.get_foreign_keys(tname, schema) + fknames_for_table[tname] = set( + [fk['name'] for fk in fkeys] + ) + for fkey in fkeys: + if tname != fkey['referred_table']: + tuples.add((fkey['referred_table'], tname)) + try: + candidate_sort = list(topological.sort(tuples, tnames)) + except exc.CircularDependencyError as err: + for edge in err.edges: + tuples.remove(edge) + remaining_fkcs.update( + (edge[1], fkc) + for fkc in fknames_for_table[edge[1]] + ) + + candidate_sort = list(topological.sort(tuples, tnames)) + return [ + (tname, fknames_for_table[tname].difference(remaining_fkcs)) + for tname in candidate_sort + ] + [(None, list(remaining_fkcs))] + def get_temp_table_names(self): """return a list of temporary table names for the current bind. diff --git a/lib/sqlalchemy/schema.py b/lib/sqlalchemy/schema.py index 95ebd05db..285ae579f 100644 --- a/lib/sqlalchemy/schema.py +++ b/lib/sqlalchemy/schema.py @@ -59,5 +59,7 @@ from .sql.ddl import ( DDLBase, DDLElement, _CreateDropBase, - _DDLCompiles + _DDLCompiles, + sort_tables, + sort_tables_and_constraints ) diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 9304bba9f..ca14c9371 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -2102,7 +2102,9 @@ class DDLCompiler(Compiled): (table.description, column.name, ce.args[0]) )) - const = self.create_table_constraints(table) + const = self.create_table_constraints( + table, _include_foreign_key_constraints= + create.include_foreign_key_constraints) if const: text += ", \n\t" + const @@ -2126,7 +2128,9 @@ class DDLCompiler(Compiled): return text - def create_table_constraints(self, table): + def create_table_constraints( + self, table, + _include_foreign_key_constraints=None): # On some DB order is significant: visit PK first, then the # other constraints (engine.ReflectionTest.testbasic failed on FB2) @@ -2134,8 +2138,15 @@ class DDLCompiler(Compiled): if table.primary_key: constraints.append(table.primary_key) + all_fkcs = table.foreign_key_constraints + if _include_foreign_key_constraints is not None: + omit_fkcs = all_fkcs.difference(_include_foreign_key_constraints) + else: + omit_fkcs = set() + constraints.extend([c for c in table._sorted_constraints - if c is not table.primary_key]) + if c is not table.primary_key and + c not in omit_fkcs]) return ", \n\t".join( p for p in @@ -2230,9 +2241,19 @@ class DDLCompiler(Compiled): self.preparer.format_sequence(drop.element) def visit_drop_constraint(self, drop): + constraint = drop.element + if constraint.name is not None: + formatted_name = self.preparer.format_constraint(constraint) + else: + formatted_name = None + + if formatted_name is None: + raise exc.CompileError( + "Can't emit DROP CONSTRAINT for constraint %r; " + "it has no name" % drop.element) return "ALTER TABLE %s DROP CONSTRAINT %s%s" % ( self.preparer.format_table(drop.element.table), - self.preparer.format_constraint(drop.element), + formatted_name, drop.cascade and " CASCADE" or "" ) diff --git a/lib/sqlalchemy/sql/ddl.py b/lib/sqlalchemy/sql/ddl.py index 534322c8d..331a283f0 100644 --- a/lib/sqlalchemy/sql/ddl.py +++ b/lib/sqlalchemy/sql/ddl.py @@ -12,7 +12,6 @@ to invoke them for a create/drop call. from .. import util from .elements import ClauseElement -from .visitors import traverse from .base import Executable, _generative, SchemaVisitor, _bind_or_error from ..util import topological from .. import event @@ -464,19 +463,28 @@ class CreateTable(_CreateDropBase): __visit_name__ = "create_table" - def __init__(self, element, on=None, bind=None): + def __init__( + self, element, on=None, bind=None, + include_foreign_key_constraints=None): """Create a :class:`.CreateTable` construct. :param element: a :class:`.Table` that's the subject of the CREATE :param on: See the description for 'on' in :class:`.DDL`. :param bind: See the description for 'bind' in :class:`.DDL`. + :param include_foreign_key_constraints: optional sequence of + :class:`.ForeignKeyConstraint` objects that will be included + inline within the CREATE construct; if omitted, all foreign key + constraints that do not specify use_alter=True are included. + + .. versionadded:: 1.0.0 """ super(CreateTable, self).__init__(element, on=on, bind=bind) self.columns = [CreateColumn(column) for column in element.columns ] + self.include_foreign_key_constraints = include_foreign_key_constraints class _DropView(_CreateDropBase): @@ -696,8 +704,10 @@ class SchemaGenerator(DDLBase): tables = self.tables else: tables = list(metadata.tables.values()) - collection = [t for t in sort_tables(tables) - if self._can_create_table(t)] + + collection = sort_tables_and_constraints( + [t for t in tables if self._can_create_table(t)]) + seq_coll = [s for s in metadata._sequences.values() if s.column is None and self._can_create_sequence(s)] @@ -709,15 +719,23 @@ class SchemaGenerator(DDLBase): for seq in seq_coll: self.traverse_single(seq, create_ok=True) - for table in collection: - self.traverse_single(table, create_ok=True) + for table, fkcs in collection: + if table is not None: + self.traverse_single( + table, create_ok=True, + include_foreign_key_constraints=fkcs) + else: + for fkc in fkcs: + self.traverse_single(fkc) metadata.dispatch.after_create(metadata, self.connection, tables=collection, checkfirst=self.checkfirst, _ddl_runner=self) - def visit_table(self, table, create_ok=False): + def visit_table( + self, table, create_ok=False, + include_foreign_key_constraints=None): if not create_ok and not self._can_create_table(table): return @@ -729,7 +747,15 @@ class SchemaGenerator(DDLBase): if column.default is not None: self.traverse_single(column.default) - self.connection.execute(CreateTable(table)) + if not self.dialect.supports_alter: + # e.g., don't omit any foreign key constraints + include_foreign_key_constraints = None + + self.connection.execute( + CreateTable( + table, + include_foreign_key_constraints=include_foreign_key_constraints + )) if hasattr(table, 'indexes'): for index in table.indexes: @@ -739,6 +765,11 @@ class SchemaGenerator(DDLBase): checkfirst=self.checkfirst, _ddl_runner=self) + def visit_foreign_key_constraint(self, constraint): + if not self.dialect.supports_alter: + return + self.connection.execute(AddConstraint(constraint)) + def visit_sequence(self, sequence, create_ok=False): if not create_ok and not self._can_create_sequence(sequence): return @@ -765,11 +796,33 @@ class SchemaDropper(DDLBase): else: tables = list(metadata.tables.values()) - collection = [ - t - for t in reversed(sort_tables(tables)) - if self._can_drop_table(t) - ] + try: + collection = reversed( + sort_tables_and_constraints( + [t for t in tables if self._can_drop_table(t)], + filter_fn= + lambda constraint: True if not self.dialect.supports_alter + else False if constraint.name is None + else None + ) + ) + except exc.CircularDependencyError as err2: + util.raise_from_cause( + exc.CircularDependencyError( + err2.message, + err2.cycles, err2.edges, + msg="Can't sort tables for DROP; an " + "unresolvable foreign key " + "dependency exists between tables: %s. Please ensure " + "that the ForeignKey and ForeignKeyConstraint objects " + "involved in the cycle have " + "names so that they can be dropped using DROP CONSTRAINT." + % ( + ", ".join(sorted([t.fullname for t in err2.cycles])) + ) + + ) + ) seq_coll = [ s @@ -781,8 +834,13 @@ class SchemaDropper(DDLBase): metadata, self.connection, tables=collection, checkfirst=self.checkfirst, _ddl_runner=self) - for table in collection: - self.traverse_single(table, drop_ok=True) + for table, fkcs in collection: + if table is not None: + self.traverse_single( + table, drop_ok=True) + else: + for fkc in fkcs: + self.traverse_single(fkc) for seq in seq_coll: self.traverse_single(seq, drop_ok=True) @@ -830,6 +888,11 @@ class SchemaDropper(DDLBase): checkfirst=self.checkfirst, _ddl_runner=self) + def visit_foreign_key_constraint(self, constraint): + if not self.dialect.supports_alter: + return + self.connection.execute(DropConstraint(constraint)) + def visit_sequence(self, sequence, drop_ok=False): if not drop_ok and not self._can_drop_sequence(sequence): return @@ -837,32 +900,159 @@ class SchemaDropper(DDLBase): def sort_tables(tables, skip_fn=None, extra_dependencies=None): - """sort a collection of Table objects in order of - their foreign-key dependency.""" + """sort a collection of :class:`.Table` objects based on dependency. - tables = list(tables) - tuples = [] - if extra_dependencies is not None: - tuples.extend(extra_dependencies) + This is a dependency-ordered sort which will emit :class:`.Table` + objects such that they will follow their dependent :class:`.Table` objects. + Tables are dependent on another based on the presence of + :class:`.ForeignKeyConstraint` objects as well as explicit dependencies + added by :meth:`.Table.add_is_dependent_on`. + + .. warning:: + + The :func:`.sort_tables` function cannot by itself accommodate + automatic resolution of dependency cycles between tables, which + are usually caused by mutually dependent foreign key constraints. + To resolve these cycles, either the + :paramref:`.ForeignKeyConstraint.use_alter` parameter may be appled + to those constraints, or use the + :func:`.sql.sort_tables_and_constraints` function which will break + out foreign key constraints involved in cycles separately. + + :param tables: a sequence of :class:`.Table` objects. + + :param skip_fn: optional callable which will be passed a + :class:`.ForeignKey` object; if it returns True, this + constraint will not be considered as a dependency. Note this is + **different** from the same parameter in + :func:`.sort_tables_and_constraints`, which is + instead passed the owning :class:`.ForeignKeyConstraint` object. + + :param extra_dependencies: a sequence of 2-tuples of tables which will + also be considered as dependent on each other. + + .. seealso:: + + :func:`.sort_tables_and_constraints` + + :meth:`.MetaData.sorted_tables` - uses this function to sort + + + """ + + if skip_fn is not None: + def _skip_fn(fkc): + for fk in fkc.elements: + if skip_fn(fk): + return True + else: + return None + else: + _skip_fn = None + + return [ + t for (t, fkcs) in + sort_tables_and_constraints( + tables, filter_fn=_skip_fn, extra_dependencies=extra_dependencies) + if t is not None + ] + + +def sort_tables_and_constraints( + tables, filter_fn=None, extra_dependencies=None): + """sort a collection of :class:`.Table` / :class:`.ForeignKeyConstraint` + objects. + + This is a dependency-ordered sort which will emit tuples of + ``(Table, [ForeignKeyConstraint, ...])`` such that each + :class:`.Table` follows its dependent :class:`.Table` objects. + Remaining :class:`.ForeignKeyConstraint` objects that are separate due to + dependency rules not satisifed by the sort are emitted afterwards + as ``(None, [ForeignKeyConstraint ...])``. + + Tables are dependent on another based on the presence of + :class:`.ForeignKeyConstraint` objects, explicit dependencies + added by :meth:`.Table.add_is_dependent_on`, as well as dependencies + stated here using the :paramref:`~.sort_tables_and_constraints.skip_fn` + and/or :paramref:`~.sort_tables_and_constraints.extra_dependencies` + parameters. + + :param tables: a sequence of :class:`.Table` objects. + + :param filter_fn: optional callable which will be passed a + :class:`.ForeignKeyConstraint` object, and returns a value based on + whether this constraint should definitely be included or excluded as + an inline constraint, or neither. If it returns False, the constraint + will definitely be included as a dependency that cannot be subject + to ALTER; if True, it will **only** be included as an ALTER result at + the end. Returning None means the constraint is included in the + table-based result unless it is detected as part of a dependency cycle. + + :param extra_dependencies: a sequence of 2-tuples of tables which will + also be considered as dependent on each other. + + .. versionadded:: 1.0.0 + + .. seealso:: + + :func:`.sort_tables` - def visit_foreign_key(fkey): - if fkey.use_alter: - return - elif skip_fn and skip_fn(fkey): - return - parent_table = fkey.column.table - if parent_table in tables: - child_table = fkey.parent.table - if parent_table is not child_table: - tuples.append((parent_table, child_table)) + """ + + fixed_dependencies = set() + mutable_dependencies = set() + + if extra_dependencies is not None: + fixed_dependencies.update(extra_dependencies) + + remaining_fkcs = set() for table in tables: - traverse(table, - {'schema_visitor': True}, - {'foreign_key': visit_foreign_key}) + for fkc in table.foreign_key_constraints: + if fkc.use_alter is True: + remaining_fkcs.add(fkc) + continue + + if filter_fn: + filtered = filter_fn(fkc) + + if filtered is True: + remaining_fkcs.add(fkc) + continue - tuples.extend( - [parent, table] for parent in table._extra_dependencies + dependent_on = fkc.referred_table + if dependent_on is not table: + mutable_dependencies.add((dependent_on, table)) + + fixed_dependencies.update( + (parent, table) for parent in table._extra_dependencies + ) + + try: + candidate_sort = list( + topological.sort( + fixed_dependencies.union(mutable_dependencies), tables + ) + ) + except exc.CircularDependencyError as err: + for edge in err.edges: + if edge in mutable_dependencies: + table = edge[1] + can_remove = [ + fkc for fkc in table.foreign_key_constraints + if filter_fn is None or filter_fn(fkc) is not False] + remaining_fkcs.update(can_remove) + for fkc in can_remove: + dependent_on = fkc.referred_table + if dependent_on is not table: + mutable_dependencies.discard((dependent_on, table)) + candidate_sort = list( + topological.sort( + fixed_dependencies.union(mutable_dependencies), tables + ) ) - return list(topological.sort(tuples, tables)) + return [ + (table, table.foreign_key_constraints.difference(remaining_fkcs)) + for table in candidate_sort + ] + [(None, list(remaining_fkcs))] diff --git a/lib/sqlalchemy/sql/schema.py b/lib/sqlalchemy/sql/schema.py index 71a0c2780..65a1da877 100644 --- a/lib/sqlalchemy/sql/schema.py +++ b/lib/sqlalchemy/sql/schema.py @@ -1476,7 +1476,14 @@ class ForeignKey(DialectKWArgs, SchemaItem): :param use_alter: passed to the underlying :class:`.ForeignKeyConstraint` to indicate the constraint should be generated/dropped externally from the CREATE TABLE/ DROP TABLE - statement. See that classes' constructor for details. + statement. See :paramref:`.ForeignKeyConstraint.use_alter` + for further description. + + .. seealso:: + + :paramref:`.ForeignKeyConstraint.use_alter` + + :ref:`use_alter` :param match: Optional string. If set, emit MATCH when issuing DDL for this constraint. Typical values include SIMPLE, PARTIAL @@ -2566,11 +2573,23 @@ class ForeignKeyConstraint(ColumnCollectionConstraint): part of the CREATE TABLE definition. Instead, generate it via an ALTER TABLE statement issued after the full collection of tables have been created, and drop it via an ALTER TABLE statement before - the full collection of tables are dropped. This is shorthand for the - usage of :class:`.AddConstraint` and :class:`.DropConstraint` - applied as "after-create" and "before-drop" events on the MetaData - object. This is normally used to generate/drop constraints on - objects that are mutually dependent on each other. + the full collection of tables are dropped. + + The use of :paramref:`.ForeignKeyConstraint.use_alter` is + particularly geared towards the case where two or more tables + are established within a mutually-dependent foreign key constraint + relationship; however, the :meth:`.MetaData.create_all` and + :meth:`.MetaData.drop_all` methods will perform this resolution + automatically, so the flag is normally not needed. + + .. versionchanged:: 1.0.0 Automatic resolution of foreign key + cycles has been added, removing the need to use the + :paramref:`.ForeignKeyConstraint.use_alter` in typical use + cases. + + .. seealso:: + + :ref:`use_alter` :param match: Optional string. If set, emit MATCH when issuing DDL for this constraint. Typical values include SIMPLE, PARTIAL @@ -2596,8 +2615,6 @@ class ForeignKeyConstraint(ColumnCollectionConstraint): self.onupdate = onupdate self.ondelete = ondelete self.link_to_name = link_to_name - if self.name is None and use_alter: - raise exc.ArgumentError("Alterable Constraint requires a name") self.use_alter = use_alter self.match = match @@ -2648,7 +2665,7 @@ class ForeignKeyConstraint(ColumnCollectionConstraint): @property def referred_table(self): """The :class:`.Table` object to which this - :class:`.ForeignKeyConstraint references. + :class:`.ForeignKeyConstraint` references. This is a dynamically calculated attribute which may not be available if the constraint and/or parent table is not yet associated with @@ -2716,15 +2733,6 @@ class ForeignKeyConstraint(ColumnCollectionConstraint): self._validate_dest_table(table) - if self.use_alter: - def supports_alter(ddl, event, schema_item, bind, **kw): - return table in set(kw['tables']) and \ - bind.dialect.supports_alter - - event.listen(table.metadata, "after_create", - ddl.AddConstraint(self, on=supports_alter)) - event.listen(table.metadata, "before_drop", - ddl.DropConstraint(self, on=supports_alter)) def copy(self, schema=None, target_table=None, **kw): fkc = ForeignKeyConstraint( @@ -3368,12 +3376,30 @@ class MetaData(SchemaItem): order in which they can be created. To get the order in which the tables would be dropped, use the ``reversed()`` Python built-in. + .. warning:: + + The :attr:`.sorted_tables` accessor cannot by itself accommodate + automatic resolution of dependency cycles between tables, which + are usually caused by mutually dependent foreign key constraints. + To resolve these cycles, either the + :paramref:`.ForeignKeyConstraint.use_alter` parameter may be appled + to those constraints, or use the + :func:`.schema.sort_tables_and_constraints` function which will break + out foreign key constraints involved in cycles separately. + .. seealso:: + :func:`.schema.sort_tables` + + :func:`.schema.sort_tables_and_constraints` + :attr:`.MetaData.tables` :meth:`.Inspector.get_table_names` + :meth:`.Inspector.get_sorted_table_and_fkc_names` + + """ return ddl.sort_tables(self.tables.values()) diff --git a/lib/sqlalchemy/testing/__init__.py b/lib/sqlalchemy/testing/__init__.py index 1f37b4b45..2375a13a9 100644 --- a/lib/sqlalchemy/testing/__init__.py +++ b/lib/sqlalchemy/testing/__init__.py @@ -23,7 +23,8 @@ from .assertions import emits_warning, emits_warning_on, uses_deprecated, \ assert_raises_message, AssertsCompiledSQL, ComparesTables, \ AssertsExecutionResults, expect_deprecated, expect_warnings -from .util import run_as_contextmanager, rowset, fail, provide_metadata, adict +from .util import run_as_contextmanager, rowset, fail, \ + provide_metadata, adict, force_drop_names crashes = skip diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py index 614a12133..646e4dea2 100644 --- a/lib/sqlalchemy/testing/plugin/plugin_base.py +++ b/lib/sqlalchemy/testing/plugin/plugin_base.py @@ -325,19 +325,11 @@ def _prep_testing_database(options, file_config): schema="test_schema") )) - for tname in reversed(inspector.get_table_names( - order_by="foreign_key")): - e.execute(schema.DropTable( - schema.Table(tname, schema.MetaData()) - )) + util.drop_all_tables(e, inspector) if config.requirements.schemas.enabled_for_config(cfg): - for tname in reversed(inspector.get_table_names( - order_by="foreign_key", schema="test_schema")): - e.execute(schema.DropTable( - schema.Table(tname, schema.MetaData(), - schema="test_schema") - )) + util.drop_all_tables(e, inspector, schema=cfg.test_schema) + util.drop_all_tables(e, inspector, schema=cfg.test_schema_2) if against(cfg, "postgresql"): from sqlalchemy.dialects import postgresql diff --git a/lib/sqlalchemy/testing/util.py b/lib/sqlalchemy/testing/util.py index 7b3f721a6..eea39b1f7 100644 --- a/lib/sqlalchemy/testing/util.py +++ b/lib/sqlalchemy/testing/util.py @@ -194,6 +194,25 @@ def provide_metadata(fn, *args, **kw): self.metadata = prev_meta +def force_drop_names(*names): + """Force the given table names to be dropped after test complete, + isolating for foreign key cycles + + """ + from . import config + from sqlalchemy import inspect + + @decorator + def go(fn, *args, **kw): + + try: + return fn(*args, **kw) + finally: + drop_all_tables( + config.db, inspect(config.db), include_names=names) + return go + + class adict(dict): """Dict keys available as attributes. Shadows.""" @@ -207,3 +226,39 @@ class adict(dict): return tuple([self[key] for key in keys]) get_all = __call__ + + +def drop_all_tables(engine, inspector, schema=None, include_names=None): + from sqlalchemy import Column, Table, Integer, MetaData, \ + ForeignKeyConstraint + from sqlalchemy.schema import DropTable, DropConstraint + + if include_names is not None: + include_names = set(include_names) + + with engine.connect() as conn: + for tname, fkcs in reversed( + inspector.get_sorted_table_and_fkc_names(schema=schema)): + if tname: + if include_names is not None and tname not in include_names: + continue + conn.execute(DropTable( + Table(tname, MetaData()) + )) + elif fkcs: + if not engine.dialect.supports_alter: + continue + for tname, fkc in fkcs: + if include_names is not None and \ + tname not in include_names: + continue + tb = Table( + tname, MetaData(), + Column('x', Integer), + Column('y', Integer), + schema=schema + ) + conn.execute(DropConstraint( + ForeignKeyConstraint( + [tb.c.x], [tb.c.y], name=fkc) + )) diff --git a/test/orm/test_cycles.py b/test/orm/test_cycles.py index 8e086ff88..fc7059dcb 100644 --- a/test/orm/test_cycles.py +++ b/test/orm/test_cycles.py @@ -284,7 +284,7 @@ class InheritTestTwo(fixtures.MappedTest): Table('c', metadata, Column('id', Integer, primary_key=True, test_needs_autoincrement=True), Column('aid', Integer, - ForeignKey('a.id', use_alter=True, name="foo"))) + ForeignKey('a.id', name="foo"))) @classmethod def setup_classes(cls): @@ -334,7 +334,7 @@ class BiDirectionalManyToOneTest(fixtures.MappedTest): Column('id', Integer, primary_key=True, test_needs_autoincrement=True), Column('data', String(30)), Column('t1id', Integer, - ForeignKey('t1.id', use_alter=True, name="foo_fk"))) + ForeignKey('t1.id', name="foo_fk"))) Table('t3', metadata, Column('id', Integer, primary_key=True, test_needs_autoincrement=True), Column('data', String(30)), @@ -436,7 +436,7 @@ class BiDirectionalOneToManyTest(fixtures.MappedTest): Table('t2', metadata, Column('c1', Integer, primary_key=True, test_needs_autoincrement=True), Column('c2', Integer, - ForeignKey('t1.c1', use_alter=True, name='t1c1_fk'))) + ForeignKey('t1.c1', name='t1c1_fk'))) @classmethod def setup_classes(cls): @@ -491,7 +491,7 @@ class BiDirectionalOneToManyTest2(fixtures.MappedTest): Table('t2', metadata, Column('c1', Integer, primary_key=True, test_needs_autoincrement=True), Column('c2', Integer, - ForeignKey('t1.c1', use_alter=True, name='t1c1_fq')), + ForeignKey('t1.c1', name='t1c1_fq')), test_needs_autoincrement=True) Table('t1_data', metadata, @@ -572,7 +572,7 @@ class OneToManyManyToOneTest(fixtures.MappedTest): Table('ball', metadata, Column('id', Integer, primary_key=True, test_needs_autoincrement=True), Column('person_id', Integer, - ForeignKey('person.id', use_alter=True, name='fk_person_id')), + ForeignKey('person.id', name='fk_person_id')), Column('data', String(30))) Table('person', metadata, @@ -1024,7 +1024,7 @@ class SelfReferentialPostUpdateTest3(fixtures.MappedTest): test_needs_autoincrement=True), Column('name', String(50), nullable=False), Column('child_id', Integer, - ForeignKey('child.id', use_alter=True, name='c1'), nullable=True)) + ForeignKey('child.id', name='c1'), nullable=True)) Table('child', metadata, Column('id', Integer, primary_key=True, @@ -1094,11 +1094,11 @@ class PostUpdateBatchingTest(fixtures.MappedTest): test_needs_autoincrement=True), Column('name', String(50), nullable=False), Column('c1_id', Integer, - ForeignKey('child1.id', use_alter=True, name='c1'), nullable=True), + ForeignKey('child1.id', name='c1'), nullable=True), Column('c2_id', Integer, - ForeignKey('child2.id', use_alter=True, name='c2'), nullable=True), + ForeignKey('child2.id', name='c2'), nullable=True), Column('c3_id', Integer, - ForeignKey('child3.id', use_alter=True, name='c3'), nullable=True) + ForeignKey('child3.id', name='c3'), nullable=True) ) Table('child1', metadata, diff --git a/test/sql/test_constraints.py b/test/sql/test_constraints.py index c0b5806ac..604b5efeb 100644 --- a/test/sql/test_constraints.py +++ b/test/sql/test_constraints.py @@ -58,8 +58,77 @@ class ConstraintGenTest(fixtures.TestBase, AssertsExecutionResults): ) ) + @testing.force_drop_names('a', 'b') + def test_fk_cant_drop_cycled_unnamed(self): + metadata = MetaData() + + Table("a", metadata, + Column('id', Integer, primary_key=True), + Column('bid', Integer), + ForeignKeyConstraint(["bid"], ["b.id"]) + ) + Table( + "b", metadata, + Column('id', Integer, primary_key=True), + Column("aid", Integer), + ForeignKeyConstraint(["aid"], ["a.id"])) + metadata.create_all(testing.db) + if testing.db.dialect.supports_alter: + assert_raises_message( + exc.CircularDependencyError, + "Can't sort tables for DROP; an unresolvable foreign key " + "dependency exists between tables: a, b. Please ensure " + "that the ForeignKey and ForeignKeyConstraint objects " + "involved in the cycle have names so that they can be " + "dropped using DROP CONSTRAINT.", + metadata.drop_all, testing.db + ) + else: + + with self.sql_execution_asserter() as asserter: + metadata.drop_all(testing.db, checkfirst=False) + + asserter.assert_( + AllOf( + CompiledSQL("DROP TABLE a"), + CompiledSQL("DROP TABLE b") + ) + ) + + @testing.provide_metadata + def test_fk_table_auto_alter_constraint_create(self): + metadata = self.metadata + + Table("a", metadata, + Column('id', Integer, primary_key=True), + Column('bid', Integer), + ForeignKeyConstraint(["bid"], ["b.id"]) + ) + Table( + "b", metadata, + Column('id', Integer, primary_key=True), + Column("aid", Integer), + ForeignKeyConstraint(["aid"], ["a.id"], name="bfk")) + self._assert_cyclic_constraint(metadata, auto=True) + + @testing.provide_metadata + def test_fk_column_auto_alter_constraint_create(self): + metadata = self.metadata + + Table("a", metadata, + Column('id', Integer, primary_key=True), + Column('bid', Integer, ForeignKey("b.id")), + ) + Table("b", metadata, + Column('id', Integer, primary_key=True), + Column("aid", Integer, + ForeignKey("a.id", name="bfk") + ), + ) + self._assert_cyclic_constraint(metadata, auto=True) + @testing.provide_metadata - def test_cyclic_fk_table_constraint_create(self): + def test_fk_table_use_alter_constraint_create(self): metadata = self.metadata Table("a", metadata, @@ -75,7 +144,7 @@ class ConstraintGenTest(fixtures.TestBase, AssertsExecutionResults): self._assert_cyclic_constraint(metadata) @testing.provide_metadata - def test_cyclic_fk_column_constraint_create(self): + def test_fk_column_use_alter_constraint_create(self): metadata = self.metadata Table("a", metadata, @@ -90,45 +159,104 @@ class ConstraintGenTest(fixtures.TestBase, AssertsExecutionResults): ) self._assert_cyclic_constraint(metadata) - def _assert_cyclic_constraint(self, metadata): - assertions = [ - CompiledSQL('CREATE TABLE b (' + def _assert_cyclic_constraint(self, metadata, auto=False): + table_assertions = [] + if auto: + if testing.db.dialect.supports_alter: + table_assertions.append( + CompiledSQL('CREATE TABLE b (' + 'id INTEGER NOT NULL, ' + 'aid INTEGER, ' + 'PRIMARY KEY (id)' + ')' + ) + ) + else: + table_assertions.append( + CompiledSQL( + 'CREATE TABLE b (' 'id INTEGER NOT NULL, ' 'aid INTEGER, ' + 'PRIMARY KEY (id), ' + 'CONSTRAINT bfk FOREIGN KEY(aid) REFERENCES a (id)' + ')' + ) + ) + + if testing.db.dialect.supports_alter: + table_assertions.append( + CompiledSQL( + 'CREATE TABLE a (' + 'id INTEGER NOT NULL, ' + 'bid INTEGER, ' 'PRIMARY KEY (id)' ')' - ), - CompiledSQL('CREATE TABLE a (' + ) + ) + else: + table_assertions.append( + CompiledSQL( + 'CREATE TABLE a (' 'id INTEGER NOT NULL, ' 'bid INTEGER, ' 'PRIMARY KEY (id), ' 'FOREIGN KEY(bid) REFERENCES b (id)' ')' - ), - ] + ) + ) + else: + table_assertions.append( + CompiledSQL('CREATE TABLE b (' + 'id INTEGER NOT NULL, ' + 'aid INTEGER, ' + 'PRIMARY KEY (id)' + ')' + ) + ) + table_assertions.append( + CompiledSQL( + 'CREATE TABLE a (' + 'id INTEGER NOT NULL, ' + 'bid INTEGER, ' + 'PRIMARY KEY (id), ' + 'FOREIGN KEY(bid) REFERENCES b (id)' + ')' + ) + ) + + assertions = [AllOf(*table_assertions)] if testing.db.dialect.supports_alter: - assertions.append( + fk_assertions = [] + fk_assertions.append( CompiledSQL('ALTER TABLE b ADD CONSTRAINT bfk ' 'FOREIGN KEY(aid) REFERENCES a (id)') ) - self.assert_sql_execution( - testing.db, - lambda: metadata.create_all(checkfirst=False), - *assertions - ) + if auto: + fk_assertions.append( + CompiledSQL('ALTER TABLE a ADD ' + 'FOREIGN KEY(bid) REFERENCES b (id)') + ) + assertions.append(AllOf(*fk_assertions)) + + with self.sql_execution_asserter() as asserter: + metadata.create_all(checkfirst=False) + asserter.assert_(*assertions) - assertions = [] if testing.db.dialect.supports_alter: - assertions.append(CompiledSQL('ALTER TABLE b DROP CONSTRAINT bfk')) - assertions.extend([ - CompiledSQL("DROP TABLE a"), - CompiledSQL("DROP TABLE b"), - ]) - self.assert_sql_execution( - testing.db, - lambda: metadata.drop_all(checkfirst=False), - *assertions - ) + assertions = [ + CompiledSQL('ALTER TABLE b DROP CONSTRAINT bfk'), + CompiledSQL("DROP TABLE a"), + CompiledSQL("DROP TABLE b") + ] + else: + assertions = [AllOf( + CompiledSQL("DROP TABLE a"), + CompiledSQL("DROP TABLE b") + )] + + with self.sql_execution_asserter() as asserter: + metadata.drop_all(checkfirst=False), + asserter.assert_(*assertions) @testing.requires.check_constraints @testing.provide_metadata @@ -542,6 +670,33 @@ class ConstraintCompilationTest(fixtures.TestBase, AssertsCompiledSQL): "REFERENCES tbl (a) MATCH SIMPLE" ) + def test_create_table_omit_fks(self): + fkcs = [ + ForeignKeyConstraint(['a'], ['remote.id'], name='foo'), + ForeignKeyConstraint(['b'], ['remote.id'], name='bar'), + ForeignKeyConstraint(['c'], ['remote.id'], name='bat'), + ] + m = MetaData() + t = Table( + 't', m, + Column('a', Integer), + Column('b', Integer), + Column('c', Integer), + *fkcs + ) + Table('remote', m, Column('id', Integer, primary_key=True)) + + self.assert_compile( + schema.CreateTable(t, include_foreign_key_constraints=[]), + "CREATE TABLE t (a INTEGER, b INTEGER, c INTEGER)" + ) + self.assert_compile( + schema.CreateTable(t, include_foreign_key_constraints=fkcs[0:2]), + "CREATE TABLE t (a INTEGER, b INTEGER, c INTEGER, " + "CONSTRAINT foo FOREIGN KEY(a) REFERENCES remote (id), " + "CONSTRAINT bar FOREIGN KEY(b) REFERENCES remote (id))" + ) + def test_deferrable_unique(self): factory = lambda **kw: UniqueConstraint('b', **kw) self._test_deferrable(factory) diff --git a/test/sql/test_ddlemit.py b/test/sql/test_ddlemit.py index 825f8228b..e191beed3 100644 --- a/test/sql/test_ddlemit.py +++ b/test/sql/test_ddlemit.py @@ -1,6 +1,6 @@ from sqlalchemy.testing import fixtures from sqlalchemy.sql.ddl import SchemaGenerator, SchemaDropper -from sqlalchemy import MetaData, Table, Column, Integer, Sequence +from sqlalchemy import MetaData, Table, Column, Integer, Sequence, ForeignKey from sqlalchemy import schema from sqlalchemy.testing.mock import Mock @@ -42,6 +42,31 @@ class EmitDDLTest(fixtures.TestBase): for i in range(1, 6) ) + def _use_alter_fixture_one(self): + m = MetaData() + + t1 = Table( + 't1', m, Column('id', Integer, primary_key=True), + Column('t2id', Integer, ForeignKey('t2.id')) + ) + t2 = Table( + 't2', m, Column('id', Integer, primary_key=True), + Column('t1id', Integer, ForeignKey('t1.id')) + ) + return m, t1, t2 + + def _fk_fixture_one(self): + m = MetaData() + + t1 = Table( + 't1', m, Column('id', Integer, primary_key=True), + Column('t2id', Integer, ForeignKey('t2.id')) + ) + t2 = Table( + 't2', m, Column('id', Integer, primary_key=True), + ) + return m, t1, t2 + def _table_seq_fixture(self): m = MetaData() @@ -172,6 +197,32 @@ class EmitDDLTest(fixtures.TestBase): self._assert_drop_tables([t1, t2, t3, t4, t5], generator, m) + def test_create_metadata_auto_alter_fk(self): + m, t1, t2 = self._use_alter_fixture_one() + generator = self._mock_create_fixture( + False, [t1, t2] + ) + self._assert_create_w_alter( + [t1, t2] + + list(t1.foreign_key_constraints) + + list(t2.foreign_key_constraints), + generator, + m + ) + + def test_create_metadata_inline_fk(self): + m, t1, t2 = self._fk_fixture_one() + generator = self._mock_create_fixture( + False, [t1, t2] + ) + self._assert_create_w_alter( + [t1, t2] + + list(t1.foreign_key_constraints) + + list(t2.foreign_key_constraints), + generator, + m + ) + def _assert_create_tables(self, elements, generator, argument): self._assert_ddl(schema.CreateTable, elements, generator, argument) @@ -188,6 +239,16 @@ class EmitDDLTest(fixtures.TestBase): (schema.DropTable, schema.DropSequence), elements, generator, argument) + def _assert_create_w_alter(self, elements, generator, argument): + self._assert_ddl( + (schema.CreateTable, schema.CreateSequence, schema.AddConstraint), + elements, generator, argument) + + def _assert_drop_w_alter(self, elements, generator, argument): + self._assert_ddl( + (schema.DropTable, schema.DropSequence, schema.DropConstraint), + elements, generator, argument) + def _assert_ddl(self, ddl_cls, elements, generator, argument): generator.traverse_single(argument) for call_ in generator.connection.execute.mock_calls: @@ -196,4 +257,8 @@ class EmitDDLTest(fixtures.TestBase): assert c.element in elements, "element %r was not expected"\ % c.element elements.remove(c.element) + if getattr(c, 'include_foreign_key_constraints', None) is not None: + elements[:] = [ + e for e in elements + if e not in set(c.include_foreign_key_constraints)] assert not elements, "elements remain in list: %r" % elements -- cgit v1.2.1 From 8582bbc5ecd65bbf1cf00f7acf2453ca23196e20 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 2 Jan 2015 10:08:21 -0500 Subject: - repair drop_all_tables --- lib/sqlalchemy/testing/plugin/plugin_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py index 646e4dea2..3563b88db 100644 --- a/lib/sqlalchemy/testing/plugin/plugin_base.py +++ b/lib/sqlalchemy/testing/plugin/plugin_base.py @@ -294,7 +294,7 @@ def _setup_requirements(argument): @post def _prep_testing_database(options, file_config): - from sqlalchemy.testing import config + from sqlalchemy.testing import config, util from sqlalchemy.testing.exclusions import against from sqlalchemy import schema, inspect -- cgit v1.2.1 From 3ab50ec0118df01d1f062458d8662bbc2200faad Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 2 Jan 2015 15:23:24 -0500 Subject: - test failures: - test_schema_2 is only on PG and doesn't need a drop all, omit this for now - py3k has exception.args[0], not message --- lib/sqlalchemy/sql/ddl.py | 2 +- lib/sqlalchemy/testing/plugin/plugin_base.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/sqlalchemy/sql/ddl.py b/lib/sqlalchemy/sql/ddl.py index 331a283f0..7a1c7fef6 100644 --- a/lib/sqlalchemy/sql/ddl.py +++ b/lib/sqlalchemy/sql/ddl.py @@ -809,7 +809,7 @@ class SchemaDropper(DDLBase): except exc.CircularDependencyError as err2: util.raise_from_cause( exc.CircularDependencyError( - err2.message, + err2.args[0], err2.cycles, err2.edges, msg="Can't sort tables for DROP; an " "unresolvable foreign key " diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py index 3563b88db..b0188aa5a 100644 --- a/lib/sqlalchemy/testing/plugin/plugin_base.py +++ b/lib/sqlalchemy/testing/plugin/plugin_base.py @@ -329,7 +329,6 @@ def _prep_testing_database(options, file_config): if config.requirements.schemas.enabled_for_config(cfg): util.drop_all_tables(e, inspector, schema=cfg.test_schema) - util.drop_all_tables(e, inspector, schema=cfg.test_schema_2) if against(cfg, "postgresql"): from sqlalchemy.dialects import postgresql -- cgit v1.2.1 From 378ad79713397183d71960b5a2c18b5b509fb137 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 2 Jan 2015 16:43:11 -0500 Subject: - put this back now that we found the source of this --- doc/build/core/constraints.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/build/core/constraints.rst b/doc/build/core/constraints.rst index a11300100..1f855c724 100644 --- a/doc/build/core/constraints.rst +++ b/doc/build/core/constraints.rst @@ -176,17 +176,13 @@ has a name. In the case of the ``'node'`` table above, we haven't named this constraint; the system will therefore attempt to emit DROP for only those constraints that are named: -.. NOTE: the parser is doing something wrong with the DROP here, - if the "DROP TABLE element" is second, the "t" is being chopped off; - it is specific to the letter "t". Look into this at some point - .. sourcecode:: pycon+sql >>> with engine.connect() as conn: ... metadata.drop_all(conn, checkfirst=False) {opensql}ALTER TABLE element DROP CONSTRAINT fk_element_parent_node_id - DROP TABLE element DROP TABLE node + DROP TABLE element {stop} -- cgit v1.2.1 From bb40221a28735cad9e5d7ece1127d9d312051c4f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 3 Jan 2015 12:10:17 -0500 Subject: - tighten the inspection in _ColumnEntity to reduce unnecessary isinstance() calls, express intent more clearly --- lib/sqlalchemy/orm/query.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index d2bdf5a00..7302574e6 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -3542,26 +3542,26 @@ class _ColumnEntity(_QueryEntity): )): self._label_name = column.key column = column._query_clause_element() - else: - self._label_name = getattr(column, 'key', None) - - if not isinstance(column, expression.ColumnElement) and \ - hasattr(column, '_select_iterable'): - for c in column._select_iterable: - if c is column: - break - _ColumnEntity(query, c, namespace=column) - else: + if isinstance(column, Bundle): + _BundleEntity(query, column) return - elif isinstance(column, Bundle): - _BundleEntity(query, column) - return + elif not isinstance(column, sql.ColumnElement): + if hasattr(column, '_select_iterable'): + # break out an object like Table into + # individual columns + for c in column._select_iterable: + if c is column: + break + _ColumnEntity(query, c, namespace=column) + else: + return - if not isinstance(column, sql.ColumnElement): raise sa_exc.InvalidRequestError( "SQL expression, column, or mapped entity " "expected - got '%r'" % (column, ) ) + else: + self._label_name = getattr(column, 'key', None) self.type = type_ = column.type if type_.hashable: -- cgit v1.2.1 From 01a22a673ecefcc212b124d11c74d99b6f02cfb0 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 3 Jan 2015 18:49:14 -0500 Subject: - clean up SET tests --- test/dialect/mysql/test_types.py | 230 +++++++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 95 deletions(-) diff --git a/test/dialect/mysql/test_types.py b/test/dialect/mysql/test_types.py index e65acc6db..546b64272 100644 --- a/test/dialect/mysql/test_types.py +++ b/test/dialect/mysql/test_types.py @@ -550,13 +550,13 @@ class TypesTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL): eq_(colspec(table.c.y5), 'y5 YEAR(4)') -class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL): +class EnumSetTest( + fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL): __only_on__ = 'mysql' __dialect__ = mysql.dialect() __backend__ = True - @testing.provide_metadata def test_enum(self): """Exercise the ENUM type.""" @@ -566,7 +566,8 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL e3 = mysql.ENUM("'a'", "'b'", strict=True) e4 = mysql.ENUM("'a'", "'b'", strict=True) - enum_table = Table('mysql_enum', self.metadata, + enum_table = Table( + 'mysql_enum', self.metadata, Column('e1', e1), Column('e2', e2, nullable=False), Column('e2generic', Enum("a", "b"), nullable=False), @@ -576,32 +577,43 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL Column('e5', mysql.ENUM("a", "b")), Column('e5generic', Enum("a", "b")), Column('e6', mysql.ENUM("'a'", "b")), - ) + ) - eq_(colspec(enum_table.c.e1), - "e1 ENUM('a','b')") - eq_(colspec(enum_table.c.e2), - "e2 ENUM('a','b') NOT NULL") - eq_(colspec(enum_table.c.e2generic), - "e2generic ENUM('a','b') NOT NULL") - eq_(colspec(enum_table.c.e3), - "e3 ENUM('a','b')") - eq_(colspec(enum_table.c.e4), - "e4 ENUM('a','b') NOT NULL") - eq_(colspec(enum_table.c.e5), - "e5 ENUM('a','b')") - eq_(colspec(enum_table.c.e5generic), - "e5generic ENUM('a','b')") - eq_(colspec(enum_table.c.e6), - "e6 ENUM('''a''','b')") + eq_( + colspec(enum_table.c.e1), + "e1 ENUM('a','b')") + eq_( + colspec(enum_table.c.e2), + "e2 ENUM('a','b') NOT NULL") + eq_( + colspec(enum_table.c.e2generic), + "e2generic ENUM('a','b') NOT NULL") + eq_( + colspec(enum_table.c.e3), + "e3 ENUM('a','b')") + eq_( + colspec(enum_table.c.e4), + "e4 ENUM('a','b') NOT NULL") + eq_( + colspec(enum_table.c.e5), + "e5 ENUM('a','b')") + eq_( + colspec(enum_table.c.e5generic), + "e5generic ENUM('a','b')") + eq_( + colspec(enum_table.c.e6), + "e6 ENUM('''a''','b')") enum_table.create() - assert_raises(exc.DBAPIError, enum_table.insert().execute, - e1=None, e2=None, e3=None, e4=None) + assert_raises( + exc.DBAPIError, enum_table.insert().execute, + e1=None, e2=None, e3=None, e4=None) - assert_raises(exc.StatementError, enum_table.insert().execute, - e1='c', e2='c', e2generic='c', e3='c', - e4='c', e5='c', e5generic='c', e6='c') + assert_raises( + exc.StatementError, + enum_table.insert().execute, + e1='c', e2='c', e2generic='c', e3='c', + e4='c', e5='c', e5generic='c', e6='c') enum_table.insert().execute() enum_table.insert().execute(e1='a', e2='a', e2generic='a', e3='a', @@ -617,67 +629,89 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL eq_(res, expected) - @testing.provide_metadata - def test_set(self): - + def _set_fixture_one(self): with testing.expect_deprecated('Manually quoting SET value literals'): e1, e2 = mysql.SET("'a'", "'b'"), mysql.SET("'a'", "'b'") e4 = mysql.SET("'a'", "b") e5 = mysql.SET("'a'", "'b'", quoting="quoted") - set_table = Table('mysql_set', self.metadata, + + set_table = Table( + 'mysql_set', self.metadata, Column('e1', e1), Column('e2', e2, nullable=False), Column('e3', mysql.SET("a", "b")), Column('e4', e4), Column('e5', e5) - ) - - eq_(colspec(set_table.c.e1), - "e1 SET('a','b')") - eq_(colspec(set_table.c.e2), - "e2 SET('a','b') NOT NULL") - eq_(colspec(set_table.c.e3), - "e3 SET('a','b')") - eq_(colspec(set_table.c.e4), - "e4 SET('''a''','b')") - eq_(colspec(set_table.c.e5), - "e5 SET('a','b')") - set_table.create() - - assert_raises(exc.DBAPIError, set_table.insert().execute, - e1=None, e2=None, e3=None, e4=None) - - if testing.against("+oursql"): - assert_raises(exc.StatementError, set_table.insert().execute, - e1='c', e2='c', e3='c', e4='c') - - set_table.insert().execute(e1='a', e2='a', e3='a', e4="'a'", e5="a,b") - set_table.insert().execute(e1='b', e2='b', e3='b', e4='b', e5="a,b") + ) + return set_table - res = set_table.select().execute().fetchall() + def test_set_colspec(self): + self.metadata = MetaData() + set_table = self._set_fixture_one() + eq_( + colspec(set_table.c.e1), + "e1 SET('a','b')") + eq_(colspec( + set_table.c.e2), + "e2 SET('a','b') NOT NULL") + eq_( + colspec(set_table.c.e3), + "e3 SET('a','b')") + eq_( + colspec(set_table.c.e4), + "e4 SET('''a''','b')") + eq_( + colspec(set_table.c.e5), + "e5 SET('a','b')") - if not testing.against("+oursql"): - # oursql receives this for first row: - # (set(['']), set(['']), set(['']), set(['']), None), - # but based on ...OS? MySQL version? not clear. - # not worth testing. + @testing.provide_metadata + def test_no_null(self): + set_table = self._set_fixture_one() + set_table.create() + assert_raises( + exc.DBAPIError, set_table.insert().execute, + e1=None, e2=None, e3=None, e4=None) - expected = [] + @testing.only_on('+oursql') + @testing.provide_metadata + def test_oursql_error_one(self): + set_table = self._set_fixture_one() + set_table.create() + assert_raises( + exc.StatementError, set_table.insert().execute, + e1='c', e2='c', e3='c', e4='c') - expected.extend([ - (set(['a']), set(['a']), set(['a']), set(["'a'"]), set(['a', 'b'])), - (set(['b']), set(['b']), set(['b']), set(['b']), set(['a', 'b'])) - ]) + @testing.provide_metadata + def test_string_roundtrip(self): + set_table = self._set_fixture_one() + set_table.create() + with testing.db.begin() as conn: + conn.execute( + set_table.insert(), + dict(e1='a', e2='a', e3='a', e4="'a'", e5="a,b")) + conn.execute( + set_table.insert(), + dict(e1='b', e2='b', e3='b', e4='b', e5="a,b")) + + expected = [ + (set(['a']), set(['a']), set(['a']), + set(["'a'"]), set(['a', 'b'])), + (set(['b']), set(['b']), set(['b']), + set(['b']), set(['a', 'b'])) + ] + res = conn.execute( + set_table.select() + ).fetchall() eq_(res, expected) @testing.provide_metadata def test_set_roundtrip_plus_reflection(self): - set_table = Table('mysql_set', self.metadata, - Column('s1', - mysql.SET("dq", "sq")), - Column('s2', mysql.SET("a")), - Column('s3', mysql.SET("5", "7", "9"))) + set_table = Table( + 'mysql_set', self.metadata, + Column('s1', mysql.SET("dq", "sq")), + Column('s2', mysql.SET("a")), + Column('s3', mysql.SET("5", "7", "9"))) eq_(colspec(set_table.c.s1), "s1 SET('dq','sq')") eq_(colspec(set_table.c.s2), "s2 SET('a')") @@ -699,29 +733,26 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL roundtrip([set(['dq']), set(['a']), set(['5'])]) roundtrip(['dq', 'a', '5'], [set(['dq']), set(['a']), set(['5'])]) - roundtrip([1, 1, 1], [set(['dq']), set(['a']), set(['5' - ])]) - roundtrip([set(['dq', 'sq']), None, set(['9', '5', '7' - ])]) - set_table.insert().execute({'s3': set(['5'])}, - {'s3': set(['5', '7'])}, {'s3': set(['5', '7', '9'])}, - {'s3': set(['7', '9'])}) - - # NOTE: the string sent to MySQL here is sensitive to ordering. - # for some reason the set ordering is always "5, 7" when we test on - # MySQLdb but in Py3K this is not guaranteed. So basically our - # SET type doesn't do ordering correctly (not sure how it can, - # as we don't know how the SET was configured in the first place.) - rows = select([set_table.c.s3], - set_table.c.s3.in_([set(['5']), ['5', '7']]) - ).execute().fetchall() + roundtrip([1, 1, 1], [set(['dq']), set(['a']), set(['5'])]) + roundtrip([set(['dq', 'sq']), None, set(['9', '5', '7'])]) + set_table.insert().execute( + {'s3': set(['5'])}, + {'s3': set(['5', '7'])}, + {'s3': set(['5', '7', '9'])}, + {'s3': set(['7', '9'])}) + + rows = select( + [set_table.c.s3], + set_table.c.s3.in_([set(['5']), ['5', '7']]) + ).execute().fetchall() found = set([frozenset(row[0]) for row in rows]) eq_(found, set([frozenset(['5']), frozenset(['5', '7'])])) @testing.provide_metadata def test_unicode_enum(self): metadata = self.metadata - t1 = Table('table', metadata, + t1 = Table( + 'table', metadata, Column('id', Integer, primary_key=True), Column('value', Enum(u('réveillé'), u('drôle'), u('S’il'))), Column('value2', mysql.ENUM(u('réveillé'), u('drôle'), u('S’il'))) @@ -731,9 +762,11 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL t1.insert().execute(value=u('réveillé'), value2=u('réveillé')) t1.insert().execute(value=u('S’il'), value2=u('S’il')) eq_(t1.select().order_by(t1.c.id).execute().fetchall(), - [(1, u('drôle'), u('drôle')), (2, u('réveillé'), u('réveillé')), - (3, u('S’il'), u('S’il'))] - ) + [ + (1, u('drôle'), u('drôle')), + (2, u('réveillé'), u('réveillé')), + (3, u('S’il'), u('S’il')) + ]) # test reflection of the enum labels @@ -743,11 +776,15 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL # TODO: what's wrong with the last element ? is there # latin-1 stuff forcing its way in ? - assert t2.c.value.type.enums[0:2] == \ - (u('réveillé'), u('drôle')) # u'S’il') # eh ? + eq_( + t2.c.value.type.enums[0:2], + (u('réveillé'), u('drôle')) # u'S’il') # eh ? + ) - assert t2.c.value2.type.enums[0:2] == \ - (u('réveillé'), u('drôle')) # u'S’il') # eh ? + eq_( + t2.c.value2.type.enums[0:2], + (u('réveillé'), u('drôle')) # u'S’il') # eh ? + ) def test_enum_compile(self): e1 = Enum('x', 'y', 'z', name='somename') @@ -767,7 +804,8 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL def test_enum_parse(self): with testing.expect_deprecated('Manually quoting ENUM value literals'): - enum_table = Table('mysql_enum', self.metadata, + enum_table = Table( + 'mysql_enum', self.metadata, Column('e1', mysql.ENUM("'a'")), Column('e2', mysql.ENUM("''")), Column('e3', mysql.ENUM('a')), @@ -795,7 +833,8 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL @testing.exclude('mysql', '<', (5,)) def test_set_parse(self): with testing.expect_deprecated('Manually quoting SET value literals'): - set_table = Table('mysql_set', self.metadata, + set_table = Table( + 'mysql_set', self.metadata, Column('e1', mysql.SET("'a'")), Column('e2', mysql.SET("''")), Column('e3', mysql.SET('a')), @@ -821,7 +860,8 @@ class EnumSetTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL eq_(t.c.e6.type.values, ("", "a")) eq_(t.c.e7.type.values, ("", "'a'", "b'b", "'")) + def colspec(c): return testing.db.dialect.ddl_compiler( - testing.db.dialect, None).get_column_specification(c) + testing.db.dialect, None).get_column_specification(c) -- cgit v1.2.1 From 93742b3d5c16d3f1e27ff0d10c8e65e1b54971a3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 3 Jan 2015 20:43:45 -0500 Subject: - The :class:`.mysql.SET` type has been overhauled to no longer assume that the empty string, or a set with a single empty string value, is in fact a set with a single empty string; instead, this is by default treated as the empty set. In order to handle persistence of a :class:`.mysql.SET` that actually wants to include the blank value ``''`` as a legitimate value, a new bitwise operational mode is added which is enabled by the :paramref:`.mysql.SET.retrieve_as_bitwise` flag, which will persist and retrieve values unambiguously using their bitflag positioning. Storage and retrieval of unicode values for driver configurations that aren't converting unicode natively is also repaired. fixes #3283 --- doc/build/changelog/changelog_10.rst | 21 +++++ doc/build/changelog/migration_10.rst | 46 ++++++++++ lib/sqlalchemy/dialects/mysql/base.py | 163 ++++++++++++++++++++++++---------- lib/sqlalchemy/util/langhelpers.py | 4 +- test/dialect/mysql/test_types.py | 120 +++++++++++++++++++++++-- 5 files changed, 295 insertions(+), 59 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 95eaff0f1..bfe2ebbc6 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,27 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, mysql + :tickets: 3283 + + The :class:`.mysql.SET` type has been overhauled to no longer + assume that the empty string, or a set with a single empty string + value, is in fact a set with a single empty string; instead, this + is by default treated as the empty set. In order to handle persistence + of a :class:`.mysql.SET` that actually wants to include the blank + value ``''`` as a legitimate value, a new bitwise operational mode + is added which is enabled by the + :paramref:`.mysql.SET.retrieve_as_bitwise` flag, which will persist + and retrieve values unambiguously using their bitflag positioning. + Storage and retrieval of unicode values for driver configurations + that aren't converting unicode natively is also repaired. + + .. seealso:: + + :ref:`change_3283` + + .. change:: :tags: feature, schema :tickets: 3282 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index f9c26017c..79756ec17 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1606,6 +1606,52 @@ by Postgresql as of 9.4. SQLAlchemy allows this using Dialect Improvements and Changes - MySQL ============================================= +.. _change_3283: + +MySQL SET Type Overhauled to support empty sets, unicode, blank value handling +------------------------------------------------------------------------------- + +The :class:`.mysql.SET` type historically not included a system of handling +blank sets and empty values separately; as different drivers had different +behaviors for treatment of empty strings and empty-string-set representations, +the SET type tried only to hedge between these behaviors, opting to treat the +empty set as ``set([''])`` as is still the current behavior for the +MySQL-Connector-Python DBAPI. +Part of the rationale here was that it was otherwise impossible to actually +store a blank string within a MySQL SET, as the driver gives us back strings +with no way to discern between ``set([''])`` and ``set()``. It was left +to the user to determine if ``set([''])`` actually meant "empty set" or not. + +The new behavior moves the use case for the blank string, which is an unusual +case that isn't even documented in MySQL's documentation, into a special +case, and the default behavior of :class:`.mysql.SET` is now: + +* to treat the empty string ``''`` as returned by MySQL-python into the empty + set ``set()``; + +* to convert the single-blank value set ``set([''])`` returned by + MySQL-Connector-Python into the empty set ``set()``; + +* To handle the case of a set type that actually wishes includes the blank + value ``''`` in its list of possible values, + a new feature (required in this use case) is implemented whereby the set + value is persisted and loaded as a bitwise integer value; the + flag :paramref:`.mysql.SET.retrieve_as_bitwise` is added in order to + enable this. + +Using the :paramref:`.mysql.SET.retrieve_as_bitwise` flag allows the set +to be persisted and retrieved with no ambiguity of values. Theoretically +this flag can be turned on in all cases, as long as the given list of +values to the type matches the ordering exactly as declared in the +database; it only makes the SQL echo output a bit more unusual. + +The default behavior of :class:`.mysql.SET` otherwise remains the same, +roundtripping values using strings. The string-based behavior now +supports unicode fully including MySQL-python with use_unicode=0. + +:ticket:`3283` + + MySQL internal "no such table" exceptions not passed to event handlers ---------------------------------------------------------------------- diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 5f990ea4e..7836e9548 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -1428,32 +1428,28 @@ class SET(_EnumeratedValues): Column('myset', SET("foo", "bar", "baz")) - :param values: The range of valid values for this SET. Values will be - quoted when generating the schema according to the quoting flag (see - below). - .. versionchanged:: 0.9.0 quoting is applied automatically to - :class:`.mysql.SET` in the same way as for :class:`.mysql.ENUM`. + The list of potential values is required in the case that this + set will be used to generate DDL for a table, or if the + :paramref:`.SET.retrieve_as_bitwise` flag is set to True. - :param charset: Optional, a column-level character set for this string - value. Takes precedence to 'ascii' or 'unicode' short-hand. + :param values: The range of valid values for this SET. - :param collation: Optional, a column-level collation for this string - value. Takes precedence to 'binary' short-hand. + :param convert_unicode: Same flag as that of + :paramref:`.String.convert_unicode`. - :param ascii: Defaults to False: short-hand for the ``latin1`` - character set, generates ASCII in schema. + :param collation: same as that of :paramref:`.String.collation` - :param unicode: Defaults to False: short-hand for the ``ucs2`` - character set, generates UNICODE in schema. + :param charset: same as that of :paramref:`.VARCHAR.charset`. - :param binary: Defaults to False: short-hand, pick the binary - collation type that matches the column's character set. Generates - BINARY in schema. This does not affect the type of data stored, - only the collation of character data. + :param ascii: same as that of :paramref:`.VARCHAR.ascii`. - :param quoting: Defaults to 'auto': automatically determine enum value - quoting. If all enum values are surrounded by the same quoting + :param unicode: same as that of :paramref:`.VARCHAR.unicode`. + + :param binary: same as that of :paramref:`.VARCHAR.binary`. + + :param quoting: Defaults to 'auto': automatically determine set value + quoting. If all values are surrounded by the same quoting character, then use 'quoted' mode. Otherwise, use 'unquoted' mode. 'quoted': values in enums are already quoted, they will be used @@ -1468,50 +1464,116 @@ class SET(_EnumeratedValues): .. versionadded:: 0.9.0 + :param retrieve_as_bitwise: if True, the data for the set type will be + persisted and selected using an integer value, where a set is coerced + into a bitwise mask for persistence. MySQL allows this mode which + has the advantage of being able to store values unambiguously, + such as the blank string ``''``. The datatype will appear + as the expression ``col + 0`` in a SELECT statement, so that the + value is coerced into an integer value in result sets. + This flag is required if one wishes + to persist a set that can store the blank string ``''`` as a value. + + .. warning:: + + When using :paramref:`.mysql.SET.retrieve_as_bitwise`, it is + essential that the list of set values is expressed in the + **exact same order** as exists on the MySQL database. + + .. versionadded:: 1.0.0 + + """ + self.retrieve_as_bitwise = kw.pop('retrieve_as_bitwise', False) values, length = self._init_values(values, kw) self.values = tuple(values) - + if not self.retrieve_as_bitwise and '' in values: + raise exc.ArgumentError( + "Can't use the blank value '' in a SET without " + "setting retrieve_as_bitwise=True") + if self.retrieve_as_bitwise: + self._bitmap = dict( + (value, 2 ** idx) + for idx, value in enumerate(self.values) + ) kw.setdefault('length', length) super(SET, self).__init__(**kw) + def column_expression(self, colexpr): + if self.retrieve_as_bitwise: + return colexpr + 0 + else: + return colexpr + def result_processor(self, dialect, coltype): - def process(value): - # The good news: - # No ',' quoting issues- commas aren't allowed in SET values - # The bad news: - # Plenty of driver inconsistencies here. - if isinstance(value, set): - # ..some versions convert '' to an empty set - if not value: - value.add('') - return value - # ...and some versions return strings - if value is not None: - return set(value.split(',')) - else: - return value + if self.retrieve_as_bitwise: + def process(value): + if value is not None: + value = int(value) + return set( + [ + elem + for idx, elem in enumerate(self.values) + if value & (2 ** idx) + ] + ) + else: + return None + else: + super_convert = super(SET, self).result_processor(dialect, coltype) + + def process(value): + if isinstance(value, util.string_types): + # MySQLdb returns a string, let's parse + if super_convert: + value = super_convert(value) + return set(re.findall(r'[^,]+', value)) + else: + # mysql-connector-python does a naive + # split(",") which throws in an empty string + if value is not None: + value.discard('') + return value return process def bind_processor(self, dialect): super_convert = super(SET, self).bind_processor(dialect) + if self.retrieve_as_bitwise: + def process(value): + if value is None: + return None + elif isinstance(value, util.int_types + util.string_types): + if super_convert: + return super_convert(value) + else: + return value + else: + int_value = 0 + for v in value: + int_value |= self._bitmap[v] + return int_value + else: - def process(value): - if value is None or isinstance( - value, util.int_types + util.string_types): - pass - else: - if None in value: - value = set(value) - value.remove(None) - value.add('') - value = ','.join(value) - if super_convert: - return super_convert(value) - else: - return value + def process(value): + # accept strings and int (actually bitflag) values directly + if value is not None and not isinstance( + value, util.int_types + util.string_types): + value = ",".join(value) + + if super_convert: + return super_convert(value) + else: + return value return process + def adapt(self, impltype, **kw): + kw['retrieve_as_bitwise'] = self.retrieve_as_bitwise + return util.constructor_copy( + self, impltype, + *self.values, + **kw + ) + # old names MSTime = TIME MSSet = SET @@ -2972,6 +3034,9 @@ class MySQLTableDefinitionParser(object): if issubclass(col_type, _EnumeratedValues): type_args = _EnumeratedValues._strip_values(type_args) + if issubclass(col_type, SET) and '' in type_args: + type_kw['retrieve_as_bitwise'] = True + type_instance = col_type(*type_args, **type_kw) col_args, col_kw = [], {} diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index ac6b50de2..7f57e501a 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -969,7 +969,7 @@ def coerce_kw_type(kw, key, type_, flexi_bool=True): kw[key] = type_(kw[key]) -def constructor_copy(obj, cls, **kw): +def constructor_copy(obj, cls, *args, **kw): """Instantiate cls using the __dict__ of obj as constructor arguments. Uses inspect to match the named arguments of ``cls``. @@ -978,7 +978,7 @@ def constructor_copy(obj, cls, **kw): names = get_cls_kwargs(cls) kw.update((k, obj.__dict__[k]) for k in names if k in obj.__dict__) - return cls(**kw) + return cls(*args, **kw) def counter(): diff --git a/test/dialect/mysql/test_types.py b/test/dialect/mysql/test_types.py index 546b64272..4e530e6b6 100644 --- a/test/dialect/mysql/test_types.py +++ b/test/dialect/mysql/test_types.py @@ -1,6 +1,6 @@ # coding: utf-8 -from sqlalchemy.testing import eq_, assert_raises +from sqlalchemy.testing import eq_, assert_raises, assert_raises_message from sqlalchemy import * from sqlalchemy import sql, exc, schema from sqlalchemy.util import u @@ -681,6 +681,67 @@ class EnumSetTest( exc.StatementError, set_table.insert().execute, e1='c', e2='c', e3='c', e4='c') + @testing.fails_on("+oursql", "oursql raises on the truncate warning") + @testing.provide_metadata + def test_empty_set_no_empty_string(self): + t = Table( + 't', self.metadata, + Column('id', Integer), + Column('data', mysql.SET("a", "b")) + ) + t.create() + with testing.db.begin() as conn: + conn.execute( + t.insert(), + {'id': 1, 'data': set()}, + {'id': 2, 'data': set([''])}, + {'id': 3, 'data': set(['a', ''])}, + {'id': 4, 'data': set(['b'])}, + ) + eq_( + conn.execute(t.select().order_by(t.c.id)).fetchall(), + [ + (1, set()), + (2, set()), + (3, set(['a'])), + (4, set(['b'])), + ] + ) + + def test_bitwise_required_for_empty(self): + assert_raises_message( + exc.ArgumentError, + "Can't use the blank value '' in a SET without setting " + "retrieve_as_bitwise=True", + mysql.SET, "a", "b", '' + ) + + @testing.provide_metadata + def test_empty_set_empty_string(self): + t = Table( + 't', self.metadata, + Column('id', Integer), + Column('data', mysql.SET("a", "b", '', retrieve_as_bitwise=True)) + ) + t.create() + with testing.db.begin() as conn: + conn.execute( + t.insert(), + {'id': 1, 'data': set()}, + {'id': 2, 'data': set([''])}, + {'id': 3, 'data': set(['a', ''])}, + {'id': 4, 'data': set(['b'])}, + ) + eq_( + conn.execute(t.select().order_by(t.c.id)).fetchall(), + [ + (1, set()), + (2, set([''])), + (3, set(['a', ''])), + (4, set(['b'])), + ] + ) + @testing.provide_metadata def test_string_roundtrip(self): set_table = self._set_fixture_one() @@ -705,6 +766,47 @@ class EnumSetTest( eq_(res, expected) + @testing.provide_metadata + def test_unicode_roundtrip(self): + set_table = Table( + 't', self.metadata, + Column('id', Integer, primary_key=True), + Column('data', mysql.SET( + u('réveillé'), u('drôle'), u('S’il'), convert_unicode=True)), + ) + + set_table.create() + with testing.db.begin() as conn: + conn.execute( + set_table.insert(), + {"data": set([u('réveillé'), u('drôle')])}) + + row = conn.execute( + set_table.select() + ).first() + + eq_( + row, + (1, set([u('réveillé'), u('drôle')])) + ) + + @testing.provide_metadata + def test_int_roundtrip(self): + set_table = self._set_fixture_one() + set_table.create() + with testing.db.begin() as conn: + conn.execute( + set_table.insert(), + dict(e1=1, e2=2, e3=3, e4=3, e5=0) + ) + res = conn.execute(set_table.select()).first() + eq_( + res, + ( + set(['a']), set(['b']), set(['a', 'b']), + set(["'a'", 'b']), set([])) + ) + @testing.provide_metadata def test_set_roundtrip_plus_reflection(self): set_table = Table( @@ -725,11 +827,11 @@ class EnumSetTest( expected = expected or store table.insert(store).execute() row = table.select().execute().first() - self.assert_(list(row) == expected) + eq_(row, tuple(expected)) table.delete().execute() roundtrip([None, None, None], [None] * 3) - roundtrip(['', '', ''], [set([''])] * 3) + roundtrip(['', '', ''], [set([])] * 3) roundtrip([set(['dq']), set(['a']), set(['5'])]) roundtrip(['dq', 'a', '5'], [set(['dq']), set(['a']), set(['5'])]) @@ -836,12 +938,14 @@ class EnumSetTest( set_table = Table( 'mysql_set', self.metadata, Column('e1', mysql.SET("'a'")), - Column('e2', mysql.SET("''")), + Column('e2', mysql.SET("''", retrieve_as_bitwise=True)), Column('e3', mysql.SET('a')), - Column('e4', mysql.SET('')), - Column('e5', mysql.SET("'a'", "''")), - Column('e6', mysql.SET("''", "'a'")), - Column('e7', mysql.SET("''", "'''a'''", "'b''b'", "''''"))) + Column('e4', mysql.SET('', retrieve_as_bitwise=True)), + Column('e5', mysql.SET("'a'", "''", retrieve_as_bitwise=True)), + Column('e6', mysql.SET("''", "'a'", retrieve_as_bitwise=True)), + Column('e7', mysql.SET( + "''", "'''a'''", "'b''b'", "''''", + retrieve_as_bitwise=True))) for col in set_table.c: self.assert_(repr(col)) -- cgit v1.2.1 From b1928d72098dd68c8aba468d94407f991f30d465 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 3 Jan 2015 23:22:07 -0500 Subject: - use a different bitwise approach here that doesn't require iterating through all possible set values --- lib/sqlalchemy/dialects/mysql/base.py | 11 ++++++----- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/langhelpers.py | 9 +++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 7836e9548..9c3f23cb2 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -1496,6 +1496,10 @@ class SET(_EnumeratedValues): (value, 2 ** idx) for idx, value in enumerate(self.values) ) + self._bitmap.update( + (2 ** idx, value) + for idx, value in enumerate(self.values) + ) kw.setdefault('length', length) super(SET, self).__init__(**kw) @@ -1510,12 +1514,9 @@ class SET(_EnumeratedValues): def process(value): if value is not None: value = int(value) + return set( - [ - elem - for idx, elem in enumerate(self.values) - if value & (2 ** idx) - ] + util.map_bits(self._bitmap.__getitem__, value) ) else: return None diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index dfed5b90a..7c85ef94b 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -36,7 +36,7 @@ from .langhelpers import iterate_attributes, class_hierarchy, \ generic_repr, counter, PluginLoader, hybridproperty, hybridmethod, \ safe_reraise,\ get_callable_argspec, only_once, attrsetter, ellipses_string, \ - warn_limited + warn_limited, map_bits from .deprecations import warn_deprecated, warn_pending_deprecation, \ deprecated, pending_deprecation, inject_docstring_text diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 7f57e501a..b708665f9 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -92,6 +92,15 @@ def _unique_symbols(used, *bases): raise NameError("exhausted namespace for symbol base %s" % base) +def map_bits(fn, n): + """Call the given function given each nonzero bit from n.""" + + while n: + b = n & (~n + 1) + yield fn(b) + n ^= b + + def decorator(target): """A signature-matching decorator factory.""" -- cgit v1.2.1 From 315db703a63f5fe5fecf6417f78ff513ff091966 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 01:53:42 -0500 Subject: - start trying to move things into __slots__. This seems to reduce the size of the many per-column objects we're hitting, but somehow the overall memory is hardly being reduced at all in initial testing --- lib/sqlalchemy/event/attr.py | 4 ++ lib/sqlalchemy/event/base.py | 2 + lib/sqlalchemy/event/registry.py | 4 ++ lib/sqlalchemy/orm/attributes.py | 97 ++++++++++++++++++++------------ lib/sqlalchemy/orm/base.py | 3 + lib/sqlalchemy/orm/interfaces.py | 4 +- lib/sqlalchemy/orm/properties.py | 1 + lib/sqlalchemy/sql/default_comparator.py | 46 ++------------- lib/sqlalchemy/sql/operators.py | 3 + lib/sqlalchemy/sql/sqltypes.py | 18 +----- lib/sqlalchemy/sql/type_api.py | 46 +++++++++++++++ test/sql/test_operators.py | 16 +++--- 12 files changed, 141 insertions(+), 103 deletions(-) diff --git a/lib/sqlalchemy/event/attr.py b/lib/sqlalchemy/event/attr.py index be2a82208..5e3499209 100644 --- a/lib/sqlalchemy/event/attr.py +++ b/lib/sqlalchemy/event/attr.py @@ -340,6 +340,8 @@ class _ListenerCollection(RefCollection, _CompoundListener): class _JoinedDispatchDescriptor(object): + __slots__ = 'name', + def __init__(self, name): self.name = name @@ -357,6 +359,8 @@ class _JoinedDispatchDescriptor(object): class _JoinedListener(_CompoundListener): _exec_once = False + __slots__ = 'parent', 'name', 'local', 'parent_listeners' + def __init__(self, parent, name, local): self.parent = parent self.name = name diff --git a/lib/sqlalchemy/event/base.py b/lib/sqlalchemy/event/base.py index 4925f6ffa..37bd2c49e 100644 --- a/lib/sqlalchemy/event/base.py +++ b/lib/sqlalchemy/event/base.py @@ -195,6 +195,8 @@ class Events(util.with_metaclass(_EventMeta, object)): class _JoinedDispatcher(object): """Represent a connection between two _Dispatch objects.""" + __slots__ = 'local', 'parent', '_parent_cls' + def __init__(self, local, parent): self.local = local self.parent = parent diff --git a/lib/sqlalchemy/event/registry.py b/lib/sqlalchemy/event/registry.py index 5b422c401..fc26f91d7 100644 --- a/lib/sqlalchemy/event/registry.py +++ b/lib/sqlalchemy/event/registry.py @@ -140,6 +140,10 @@ class _EventKey(object): """Represent :func:`.listen` arguments. """ + __slots__ = ( + 'target', 'identifier', 'fn', 'fn_key', 'fn_wrap', 'dispatch_target' + ) + def __init__(self, target, identifier, fn, dispatch_target, _fn_wrap=None): self.target = target diff --git a/lib/sqlalchemy/orm/attributes.py b/lib/sqlalchemy/orm/attributes.py index 2b4c3ec75..e9c8c511a 100644 --- a/lib/sqlalchemy/orm/attributes.py +++ b/lib/sqlalchemy/orm/attributes.py @@ -345,18 +345,16 @@ class Event(object): .. versionadded:: 0.9.0 - """ - - impl = None - """The :class:`.AttributeImpl` which is the current event initiator. - """ + :var impl: The :class:`.AttributeImpl` which is the current event + initiator. - op = None - """The symbol :attr:`.OP_APPEND`, :attr:`.OP_REMOVE` or :attr:`.OP_REPLACE`, - indicating the source operation. + :var op: The symbol :attr:`.OP_APPEND`, :attr:`.OP_REMOVE` or + :attr:`.OP_REPLACE`, indicating the source operation. """ + __slots__ = 'impl', 'op', 'parent_token' + def __init__(self, attribute_impl, op): self.impl = attribute_impl self.op = op @@ -455,6 +453,11 @@ class AttributeImpl(object): self.expire_missing = expire_missing + __slots__ = ( + 'class_', 'key', 'callable_', 'dispatch', 'trackparent', + 'parent_token', 'send_modified_events', 'is_equal', 'expire_missing' + ) + def __str__(self): return "%s.%s" % (self.class_.__name__, self.key) @@ -654,6 +657,23 @@ class ScalarAttributeImpl(AttributeImpl): supports_population = True collection = False + __slots__ = '_replace_token', '_append_token', '_remove_token' + + def __init__(self, *arg, **kw): + super(ScalarAttributeImpl, self).__init__(*arg, **kw) + self._replace_token = self._append_token = None + self._remove_token = None + + def _init_append_token(self): + self._replace_token = self._append_token = Event(self, OP_REPLACE) + return self._replace_token + + _init_append_or_replace_token = _init_append_token + + def _init_remove_token(self): + self._remove_token = Event(self, OP_REMOVE) + return self._remove_token + def delete(self, state, dict_): # TODO: catch key errors, convert to attributeerror? @@ -692,27 +712,18 @@ class ScalarAttributeImpl(AttributeImpl): state._modified_event(dict_, self, old) dict_[self.key] = value - @util.memoized_property - def _replace_token(self): - return Event(self, OP_REPLACE) - - @util.memoized_property - def _append_token(self): - return Event(self, OP_REPLACE) - - @util.memoized_property - def _remove_token(self): - return Event(self, OP_REMOVE) - def fire_replace_event(self, state, dict_, value, previous, initiator): for fn in self.dispatch.set: value = fn( - state, value, previous, initiator or self._replace_token) + state, value, previous, + initiator or self._replace_token or + self._init_append_or_replace_token()) return value def fire_remove_event(self, state, dict_, value, initiator): for fn in self.dispatch.remove: - fn(state, value, initiator or self._remove_token) + fn(state, value, + initiator or self._remove_token or self._init_remove_token()) @property def type(self): @@ -732,9 +743,13 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): supports_population = True collection = False + __slots__ = () + def delete(self, state, dict_): old = self.get(state, dict_) - self.fire_remove_event(state, dict_, old, self._remove_token) + self.fire_remove_event( + state, dict_, old, + self._remove_token or self._init_remove_token()) del dict_[self.key] def get_history(self, state, dict_, passive=PASSIVE_OFF): @@ -807,7 +822,8 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): self.sethasparent(instance_state(value), state, False) for fn in self.dispatch.remove: - fn(state, value, initiator or self._remove_token) + fn(state, value, initiator or + self._remove_token or self._init_remove_token()) state._modified_event(dict_, self, value) @@ -819,7 +835,8 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): for fn in self.dispatch.set: value = fn( - state, value, previous, initiator or self._replace_token) + state, value, previous, initiator or + self._replace_token or self._init_append_or_replace_token()) state._modified_event(dict_, self, previous) @@ -846,6 +863,8 @@ class CollectionAttributeImpl(AttributeImpl): supports_population = True collection = True + __slots__ = 'copy', 'collection_factory', '_append_token', '_remove_token' + def __init__(self, class_, key, callable_, dispatch, typecallable=None, trackparent=False, extension=None, copy_function=None, compare_function=None, **kwargs): @@ -862,6 +881,8 @@ class CollectionAttributeImpl(AttributeImpl): copy_function = self.__copy self.copy = copy_function self.collection_factory = typecallable + self._append_token = None + self._remove_token = None if getattr(self.collection_factory, "_sa_linker", None): @@ -873,6 +894,14 @@ class CollectionAttributeImpl(AttributeImpl): def unlink(target, collection, collection_adapter): collection._sa_linker(None) + def _init_append_token(self): + self._append_token = Event(self, OP_APPEND) + return self._append_token + + def _init_remove_token(self): + self._remove_token = Event(self, OP_REMOVE) + return self._remove_token + def __copy(self, item): return [y for y in collections.collection_adapter(item)] @@ -915,17 +944,11 @@ class CollectionAttributeImpl(AttributeImpl): return [(instance_state(o), o) for o in current] - @util.memoized_property - def _append_token(self): - return Event(self, OP_APPEND) - - @util.memoized_property - def _remove_token(self): - return Event(self, OP_REMOVE) - def fire_append_event(self, state, dict_, value, initiator): for fn in self.dispatch.append: - value = fn(state, value, initiator or self._append_token) + value = fn( + state, value, + initiator or self._append_token or self._init_append_token()) state._modified_event(dict_, self, NEVER_SET, True) @@ -942,7 +965,8 @@ class CollectionAttributeImpl(AttributeImpl): self.sethasparent(instance_state(value), state, False) for fn in self.dispatch.remove: - fn(state, value, initiator or self._remove_token) + fn(state, value, + initiator or self._remove_token or self._init_remove_token()) state._modified_event(dict_, self, NEVER_SET, True) @@ -1134,7 +1158,8 @@ def backref_listeners(attribute, key, uselist): impl.pop(old_state, old_dict, state.obj(), - parent_impl._append_token, + parent_impl._append_token or + parent_impl._init_append_token(), passive=PASSIVE_NO_FETCH) if child is not None: diff --git a/lib/sqlalchemy/orm/base.py b/lib/sqlalchemy/orm/base.py index 3390ceec4..afeeba322 100644 --- a/lib/sqlalchemy/orm/base.py +++ b/lib/sqlalchemy/orm/base.py @@ -438,6 +438,8 @@ class InspectionAttr(object): """ + __slots__ = () + is_selectable = False """Return True if this object is an instance of :class:`.Selectable`.""" @@ -520,3 +522,4 @@ class _MappedAttribute(object): attributes. """ + __slots__ = () diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index ad2452c1b..bff73258c 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -303,6 +303,8 @@ class PropComparator(operators.ColumnOperators): """ + __slots__ = 'prop', 'property', '_parentmapper', '_adapt_to_entity' + def __init__(self, prop, parentmapper, adapt_to_entity=None): self.prop = self.property = prop self._parentmapper = parentmapper @@ -331,7 +333,7 @@ class PropComparator(operators.ColumnOperators): else: return self._adapt_to_entity._adapt_element - @util.memoized_property + @property def info(self): return self.property.info diff --git a/lib/sqlalchemy/orm/properties.py b/lib/sqlalchemy/orm/properties.py index 62ea93fb3..291fabdd0 100644 --- a/lib/sqlalchemy/orm/properties.py +++ b/lib/sqlalchemy/orm/properties.py @@ -224,6 +224,7 @@ class ColumnProperty(StrategizedProperty): :attr:`.TypeEngine.comparator_factory` """ + @util.memoized_instancemethod def __clause_element__(self): if self.adapter: diff --git a/lib/sqlalchemy/sql/default_comparator.py b/lib/sqlalchemy/sql/default_comparator.py index d26fdc455..c898b78d6 100644 --- a/lib/sqlalchemy/sql/default_comparator.py +++ b/lib/sqlalchemy/sql/default_comparator.py @@ -9,8 +9,8 @@ """ from .. import exc, util -from . import operators from . import type_api +from . import operators from .elements import BindParameter, True_, False_, BinaryExpression, \ Null, _const_expr, _clause_element_as_expr, \ ClauseList, ColumnElement, TextClause, UnaryExpression, \ @@ -18,7 +18,7 @@ from .elements import BindParameter, True_, False_, BinaryExpression, \ from .selectable import SelectBase, Alias, Selectable, ScalarSelect -class _DefaultColumnComparator(operators.ColumnOperators): +class _DefaultColumnComparator(object): """Defines comparison and math operations. See :class:`.ColumnOperators` and :class:`.Operators` for descriptions @@ -26,46 +26,6 @@ class _DefaultColumnComparator(operators.ColumnOperators): """ - @util.memoized_property - def type(self): - return self.expr.type - - def operate(self, op, *other, **kwargs): - o = self.operators[op.__name__] - return o[0](self, self.expr, op, *(other + o[1:]), **kwargs) - - def reverse_operate(self, op, other, **kwargs): - o = self.operators[op.__name__] - return o[0](self, self.expr, op, other, - reverse=True, *o[1:], **kwargs) - - def _adapt_expression(self, op, other_comparator): - """evaluate the return type of , - and apply any adaptations to the given operator. - - This method determines the type of a resulting binary expression - given two source types and an operator. For example, two - :class:`.Column` objects, both of the type :class:`.Integer`, will - produce a :class:`.BinaryExpression` that also has the type - :class:`.Integer` when compared via the addition (``+``) operator. - However, using the addition operator with an :class:`.Integer` - and a :class:`.Date` object will produce a :class:`.Date`, assuming - "days delta" behavior by the database (in reality, most databases - other than Postgresql don't accept this particular operation). - - The method returns a tuple of the form , . - The resulting operator and type will be those applied to the - resulting :class:`.BinaryExpression` as the final operator and the - right-hand side of the expression. - - Note that only a subset of operators make usage of - :meth:`._adapt_expression`, - including math operators and user-defined operators, but not - boolean comparison or special SQL keywords like MATCH or BETWEEN. - - """ - return op, other_comparator.type - def _boolean_compare(self, expr, op, obj, negate=None, reverse=False, _python_is_types=(util.NoneType, bool), result_type = None, @@ -320,3 +280,5 @@ class _DefaultColumnComparator(operators.ColumnOperators): return expr._bind_param(operator, other) else: return other + +the_comparator = _DefaultColumnComparator() diff --git a/lib/sqlalchemy/sql/operators.py b/lib/sqlalchemy/sql/operators.py index a328b023e..f71cba913 100644 --- a/lib/sqlalchemy/sql/operators.py +++ b/lib/sqlalchemy/sql/operators.py @@ -38,6 +38,7 @@ class Operators(object): :class:`.ColumnOperators`. """ + __slots__ = () def __and__(self, other): """Implement the ``&`` operator. @@ -267,6 +268,8 @@ class ColumnOperators(Operators): """ + __slots__ = () + timetuple = None """Hack, allows datetime objects to be compared on the LHS.""" diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 7bb5c5515..9b0d26601 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -14,7 +14,6 @@ import codecs from .type_api import TypeEngine, TypeDecorator, to_instance from .elements import quoted_name, type_coerce, _defer_name -from .default_comparator import _DefaultColumnComparator from .. import exc, util, processors from .base import _bind_or_error, SchemaEventTarget from . import operators @@ -1704,19 +1703,4 @@ type_api.NULLTYPE = NULLTYPE type_api.MATCHTYPE = MATCHTYPE type_api._type_map = _type_map -# this one, there's all kinds of ways to play it, but at the EOD -# there's just a giant dependency cycle between the typing system and -# the expression element system, as you might expect. We can use -# importlaters or whatnot, but the typing system just necessarily has -# to have some kind of connection like this. right now we're injecting the -# _DefaultColumnComparator implementation into the TypeEngine.Comparator -# interface. Alternatively TypeEngine.Comparator could have an "impl" -# injected, though just injecting the base is simpler, error free, and more -# performant. - - -class Comparator(_DefaultColumnComparator): - BOOLEANTYPE = BOOLEANTYPE - -TypeEngine.Comparator.__bases__ = ( - Comparator, ) + TypeEngine.Comparator.__bases__ +TypeEngine.Comparator.BOOLEANTYPE = BOOLEANTYPE diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py index 03fed3878..834640928 100644 --- a/lib/sqlalchemy/sql/type_api.py +++ b/lib/sqlalchemy/sql/type_api.py @@ -21,6 +21,7 @@ NULLTYPE = None STRINGTYPE = None MATCHTYPE = None + class TypeEngine(Visitable): """The ultimate base class for all SQL datatypes. @@ -45,9 +46,51 @@ class TypeEngine(Visitable): """ + __slots__ = 'expr', 'type' def __init__(self, expr): self.expr = expr + self.type = expr.type + + @util.dependencies('sqlalchemy.sql.default_comparator') + def operate(self, default_comparator, op, *other, **kwargs): + comp = default_comparator.the_comparator + o = comp.operators[op.__name__] + return o[0](comp, self.expr, op, *(other + o[1:]), **kwargs) + + @util.dependencies('sqlalchemy.sql.default_comparator') + def reverse_operate(self, default_comparator, op, other, **kwargs): + comp = default_comparator.the_comparator + o = comp.operators[op.__name__] + return o[0](comp, self.expr, op, other, + reverse=True, *o[1:], **kwargs) + + def _adapt_expression(self, op, other_comparator): + """evaluate the return type of , + and apply any adaptations to the given operator. + + This method determines the type of a resulting binary expression + given two source types and an operator. For example, two + :class:`.Column` objects, both of the type :class:`.Integer`, will + produce a :class:`.BinaryExpression` that also has the type + :class:`.Integer` when compared via the addition (``+``) operator. + However, using the addition operator with an :class:`.Integer` + and a :class:`.Date` object will produce a :class:`.Date`, assuming + "days delta" behavior by the database (in reality, most databases + other than Postgresql don't accept this particular operation). + + The method returns a tuple of the form , . + The resulting operator and type will be those applied to the + resulting :class:`.BinaryExpression` as the final operator and the + right-hand side of the expression. + + Note that only a subset of operators make usage of + :meth:`._adapt_expression`, + including math operators and user-defined operators, but not + boolean comparison or special SQL keywords like MATCH or BETWEEN. + + """ + return op, other_comparator.type def __reduce__(self): return _reconstitute_comparator, (self.expr, ) @@ -454,6 +497,8 @@ class UserDefinedType(TypeEngine): __visit_name__ = "user_defined" class Comparator(TypeEngine.Comparator): + __slots__ = () + def _adapt_expression(self, op, other_comparator): if hasattr(self.type, 'adapt_operator'): util.warn_deprecated( @@ -617,6 +662,7 @@ class TypeDecorator(TypeEngine): """ class Comparator(TypeEngine.Comparator): + __slots__ = () def operate(self, op, *other, **kwargs): kwargs['_python_is_types'] = self.expr.type.coerce_to_is_types diff --git a/test/sql/test_operators.py b/test/sql/test_operators.py index 3b8b20513..0985020d1 100644 --- a/test/sql/test_operators.py +++ b/test/sql/test_operators.py @@ -361,7 +361,7 @@ class CustomComparatorTest(_CustomComparatorTests, fixtures.TestBase): class comparator_factory(TypeEngine.Comparator): def __init__(self, expr): - self.expr = expr + super(MyInteger.comparator_factory, self).__init__(expr) def __add__(self, other): return self.expr.op("goofy")(other) @@ -382,7 +382,7 @@ class TypeDecoratorComparatorTest(_CustomComparatorTests, fixtures.TestBase): class comparator_factory(TypeDecorator.Comparator): def __init__(self, expr): - self.expr = expr + super(MyInteger.comparator_factory, self).__init__(expr) def __add__(self, other): return self.expr.op("goofy")(other) @@ -404,7 +404,7 @@ class TypeDecoratorTypeDecoratorComparatorTest( class comparator_factory(TypeDecorator.Comparator): def __init__(self, expr): - self.expr = expr + super(MyIntegerOne.comparator_factory, self).__init__(expr) def __add__(self, other): return self.expr.op("goofy")(other) @@ -429,7 +429,9 @@ class TypeDecoratorWVariantComparatorTest( class comparator_factory(TypeEngine.Comparator): def __init__(self, expr): - self.expr = expr + super( + SomeOtherInteger.comparator_factory, + self).__init__(expr) def __add__(self, other): return self.expr.op("not goofy")(other) @@ -443,7 +445,7 @@ class TypeDecoratorWVariantComparatorTest( class comparator_factory(TypeDecorator.Comparator): def __init__(self, expr): - self.expr = expr + super(MyInteger.comparator_factory, self).__init__(expr) def __add__(self, other): return self.expr.op("goofy")(other) @@ -464,7 +466,7 @@ class CustomEmbeddedinTypeDecoratorTest( class comparator_factory(TypeEngine.Comparator): def __init__(self, expr): - self.expr = expr + super(MyInteger.comparator_factory, self).__init__(expr) def __add__(self, other): return self.expr.op("goofy")(other) @@ -486,7 +488,7 @@ class NewOperatorTest(_CustomComparatorTests, fixtures.TestBase): class comparator_factory(TypeEngine.Comparator): def __init__(self, expr): - self.expr = expr + super(MyInteger.comparator_factory, self).__init__(expr) def foob(self, other): return self.expr.op("foob")(other) -- cgit v1.2.1 From cf272325635c1205da7fd2668eac3c8ac409dafb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 15:07:36 -0500 Subject: - wip - start factoring events so that we aren't using descriptors for dispatch, allowing us to move to __slots__ --- lib/sqlalchemy/event/attr.py | 75 ++++++++------------ lib/sqlalchemy/event/base.py | 146 ++++++++++++++++++++++++++------------- lib/sqlalchemy/event/legacy.py | 46 ++++++------ lib/sqlalchemy/event/registry.py | 12 ++-- lib/sqlalchemy/orm/events.py | 3 +- lib/sqlalchemy/orm/mapper.py | 4 +- lib/sqlalchemy/pool.py | 1 + test/base/test_events.py | 21 ++---- 8 files changed, 166 insertions(+), 142 deletions(-) diff --git a/lib/sqlalchemy/event/attr.py b/lib/sqlalchemy/event/attr.py index 5e3499209..de5d34950 100644 --- a/lib/sqlalchemy/event/attr.py +++ b/lib/sqlalchemy/event/attr.py @@ -46,11 +46,11 @@ class RefCollection(object): return weakref.ref(self, registry._collection_gced) -class _DispatchDescriptor(RefCollection): - """Class-level attributes on :class:`._Dispatch` classes.""" +class _ClsLevelDispatch(RefCollection): + """Class-level events on :class:`._Dispatch` classes.""" def __init__(self, parent_dispatch_cls, fn): - self.__name__ = fn.__name__ + self.name = fn.__name__ argspec = util.inspect_getargspec(fn) self.arg_names = argspec.args[1:] self.has_kw = bool(argspec.keywords) @@ -64,7 +64,6 @@ class _DispatchDescriptor(RefCollection): self, parent_dispatch_cls, fn) self._clslevel = weakref.WeakKeyDictionary() - self._empty_listeners = weakref.WeakKeyDictionary() def _adjust_fn_spec(self, fn, named): if named: @@ -152,34 +151,23 @@ class _DispatchDescriptor(RefCollection): def for_modify(self, obj): """Return an event collection which can be modified. - For _DispatchDescriptor at the class level of + For _ClsLevelDispatch at the class level of a dispatcher, this returns self. """ return self - def __get__(self, obj, cls): - if obj is None: - return self - elif obj._parent_cls in self._empty_listeners: - ret = self._empty_listeners[obj._parent_cls] - else: - self._empty_listeners[obj._parent_cls] = ret = \ - _EmptyListener(self, obj._parent_cls) - # assigning it to __dict__ means - # memoized for fast re-access. but more memory. - obj.__dict__[self.__name__] = ret - return ret +class _InstanceLevelDispatch(object): + __slots__ = () -class _HasParentDispatchDescriptor(object): def _adjust_fn_spec(self, fn, named): return self.parent._adjust_fn_spec(fn, named) -class _EmptyListener(_HasParentDispatchDescriptor): - """Serves as a class-level interface to the events - served by a _DispatchDescriptor, when there are no +class _EmptyListener(_InstanceLevelDispatch): + """Serves as a proxy interface to the events + served by a _ClsLevelDispatch, when there are no instance-level events present. Is replaced by _ListenerCollection when instance-level @@ -187,14 +175,17 @@ class _EmptyListener(_HasParentDispatchDescriptor): """ + propagate = frozenset() + listeners = () + + __slots__ = 'parent', 'parent_listeners', 'name' + def __init__(self, parent, target_cls): if target_cls not in parent._clslevel: parent.update_subclass(target_cls) - self.parent = parent # _DispatchDescriptor + self.parent = parent # _ClsLevelDispatch self.parent_listeners = parent._clslevel[target_cls] - self.name = parent.__name__ - self.propagate = frozenset() - self.listeners = () + self.name = parent.name def for_modify(self, obj): """Return an event collection which can be modified. @@ -205,9 +196,11 @@ class _EmptyListener(_HasParentDispatchDescriptor): and returns it. """ - result = _ListenerCollection(self.parent, obj._parent_cls) - if obj.__dict__[self.name] is self: - obj.__dict__[self.name] = result + result = _ListenerCollection(self.parent, obj._instance_cls) + if getattr(obj, self.name) is self: + setattr(obj, self.name, result) + else: + assert isinstance(getattr(obj, self.name), _JoinedListener) return result def _needs_modify(self, *args, **kw): @@ -233,9 +226,11 @@ class _EmptyListener(_HasParentDispatchDescriptor): __nonzero__ = __bool__ -class _CompoundListener(_HasParentDispatchDescriptor): +class _CompoundListener(_InstanceLevelDispatch): _exec_once = False + __slots__ = () + @util.memoized_property def _exec_once_mutex(self): return threading.Lock() @@ -282,12 +277,15 @@ class _ListenerCollection(RefCollection, _CompoundListener): """ + # RefCollection has a @memoized_property, so can't do + # __slots__ here + def __init__(self, parent, target_cls): if target_cls not in parent._clslevel: parent.update_subclass(target_cls) self.parent_listeners = parent._clslevel[target_cls] self.parent = parent - self.name = parent.__name__ + self.name = parent.name self.listeners = collections.deque() self.propagate = set() @@ -339,23 +337,6 @@ class _ListenerCollection(RefCollection, _CompoundListener): self.listeners.clear() -class _JoinedDispatchDescriptor(object): - __slots__ = 'name', - - def __init__(self, name): - self.name = name - - def __get__(self, obj, cls): - if obj is None: - return self - else: - obj.__dict__[self.name] = ret = _JoinedListener( - obj.parent, self.name, - getattr(obj.local, self.name) - ) - return ret - - class _JoinedListener(_CompoundListener): _exec_once = False diff --git a/lib/sqlalchemy/event/base.py b/lib/sqlalchemy/event/base.py index 37bd2c49e..962d850c2 100644 --- a/lib/sqlalchemy/event/base.py +++ b/lib/sqlalchemy/event/base.py @@ -17,9 +17,11 @@ instances of ``_Dispatch``. """ from __future__ import absolute_import +import weakref + from .. import util -from .attr import _JoinedDispatchDescriptor, \ - _EmptyListener, _DispatchDescriptor +from .attr import _JoinedListener, \ + _EmptyListener, _ClsLevelDispatch _registrars = util.defaultdict(list) @@ -34,10 +36,11 @@ class _UnpickleDispatch(object): """ - def __call__(self, _parent_cls): - for cls in _parent_cls.__mro__: + def __call__(self, _instance_cls): + for cls in _instance_cls.__mro__: if 'dispatch' in cls.__dict__: - return cls.__dict__['dispatch'].dispatch_cls(_parent_cls) + return cls.__dict__['dispatch'].\ + dispatch_cls._for_class(_instance_cls) else: raise AttributeError("No class with a 'dispatch' member present.") @@ -62,16 +65,41 @@ class _Dispatch(object): """ - _events = None - """reference the :class:`.Events` class which this - :class:`._Dispatch` is created for.""" - - def __init__(self, _parent_cls): - self._parent_cls = _parent_cls - - @util.classproperty - def _listen(cls): - return cls._events._listen + # in one ORM edge case, an attribute is added to _Dispatch, + # so __dict__ is used in just that case and potentially others. + __slots__ = '_parent', '_instance_cls', '__dict__' + + _empty_listeners = weakref.WeakKeyDictionary() + + def __init__(self, parent, instance_cls=None): + self._parent = parent + self._instance_cls = instance_cls + if instance_cls: + try: + _empty_listeners = self._empty_listeners[instance_cls] + except KeyError: + _empty_listeners = self._empty_listeners[instance_cls] = [ + _EmptyListener(ls, instance_cls) + for ls in parent._event_descriptors + ] + for ls in _empty_listeners: + setattr(self, ls.name, ls) + + @property + def _event_descriptors(self): + for k in self._event_names: + yield getattr(self, k) + + def _for_class(self, instance_cls): + return self.__class__(self, instance_cls) + + def _for_instance(self, instance): + instance_cls = instance.__class__ + return self._for_class(instance_cls) + + @property + def _listen(self): + return self._events._listen def _join(self, other): """Create a 'join' of this :class:`._Dispatch` and another. @@ -83,36 +111,27 @@ class _Dispatch(object): if '_joined_dispatch_cls' not in self.__class__.__dict__: cls = type( "Joined%s" % self.__class__.__name__, - (_JoinedDispatcher, self.__class__), {} + (_JoinedDispatcher, ), {'__slots__': self._event_names} ) - for ls in _event_descriptors(self): - setattr(cls, ls.name, _JoinedDispatchDescriptor(ls.name)) self.__class__._joined_dispatch_cls = cls return self._joined_dispatch_cls(self, other) def __reduce__(self): - return _UnpickleDispatch(), (self._parent_cls, ) + return _UnpickleDispatch(), (self._instance_cls, ) def _update(self, other, only_propagate=True): """Populate from the listeners in another :class:`_Dispatch` object.""" - - for ls in _event_descriptors(other): + for ls in other._event_descriptors: if isinstance(ls, _EmptyListener): continue getattr(self, ls.name).\ for_modify(self)._update(ls, only_propagate=only_propagate) - @util.hybridmethod def _clear(self): - for attr in dir(self): - if _is_event_name(attr): - getattr(self, attr).for_modify(self).clear() - - -def _event_descriptors(target): - return [getattr(target, k) for k in dir(target) if _is_event_name(k)] + for ls in self._event_descriptors: + ls.for_modify(self).clear() class _EventMeta(type): @@ -131,26 +150,37 @@ def _create_dispatcher_class(cls, classname, bases, dict_): # there's all kinds of ways to do this, # i.e. make a Dispatch class that shares the '_listen' method # of the Event class, this is the straight monkeypatch. - dispatch_base = getattr(cls, 'dispatch', _Dispatch) + if hasattr(cls, 'dispatch'): + dispatch_base = cls.dispatch.__class__ + else: + dispatch_base = _Dispatch + + event_names = [k for k in dict_ if _is_event_name(k)] dispatch_cls = type("%sDispatch" % classname, - (dispatch_base, ), {}) - cls._set_dispatch(cls, dispatch_cls) + (dispatch_base, ), {'__slots__': event_names}) + + dispatch_cls._event_names = event_names - for k in dict_: - if _is_event_name(k): - setattr(dispatch_cls, k, _DispatchDescriptor(cls, dict_[k])) - _registrars[k].append(cls) + dispatch_inst = cls._set_dispatch(cls, dispatch_cls) + for k in dispatch_cls._event_names: + setattr(dispatch_inst, k, _ClsLevelDispatch(cls, dict_[k])) + _registrars[k].append(cls) + + for super_ in dispatch_cls.__bases__: + if issubclass(super_, _Dispatch) and super_ is not _Dispatch: + for ls in super_._events.dispatch._event_descriptors: + setattr(dispatch_inst, ls.name, ls) + dispatch_cls._event_names.append(ls.name) if getattr(cls, '_dispatch_target', None): cls._dispatch_target.dispatch = dispatcher(cls) def _remove_dispatcher(cls): - for k in dir(cls): - if _is_event_name(k): - _registrars[k].remove(cls) - if not _registrars[k]: - del _registrars[k] + for k in cls.dispatch._event_names: + _registrars[k].remove(cls) + if not _registrars[k]: + del _registrars[k] class Events(util.with_metaclass(_EventMeta, object)): @@ -163,17 +193,30 @@ class Events(util.with_metaclass(_EventMeta, object)): # "self.dispatch._events." # @staticemethod to allow easy "super" calls while in a metaclass # constructor. - cls.dispatch = dispatch_cls + cls.dispatch = dispatch_cls(None) dispatch_cls._events = cls + return cls.dispatch @classmethod def _accept_with(cls, target): # Mapper, ClassManager, Session override this to # also accept classes, scoped_sessions, sessionmakers, etc. if hasattr(target, 'dispatch') and ( - isinstance(target.dispatch, cls.dispatch) or - isinstance(target.dispatch, type) and - issubclass(target.dispatch, cls.dispatch) + + isinstance(target.dispatch, cls.dispatch.__class__) or + + + ( + isinstance(target.dispatch, type) and + isinstance(target.dispatch, cls.dispatch.__class__) + ) or + + ( + isinstance(target.dispatch, _JoinedDispatcher) and + isinstance(target.dispatch.parent, cls.dispatch.__class__) + ) + + ): return target else: @@ -195,12 +238,19 @@ class Events(util.with_metaclass(_EventMeta, object)): class _JoinedDispatcher(object): """Represent a connection between two _Dispatch objects.""" - __slots__ = 'local', 'parent', '_parent_cls' + __slots__ = 'local', 'parent', '_instance_cls' def __init__(self, local, parent): self.local = local self.parent = parent - self._parent_cls = local._parent_cls + self._instance_cls = self.local._instance_cls + for ls in local._event_descriptors: + setattr(self, ls.name, _JoinedListener( + parent, ls.name, ls)) + + @property + def _listen(self): + return self.parent._listen class dispatcher(object): @@ -218,5 +268,5 @@ class dispatcher(object): def __get__(self, obj, cls): if obj is None: return self.dispatch_cls - obj.__dict__['dispatch'] = disp = self.dispatch_cls(cls) + obj.__dict__['dispatch'] = disp = self.dispatch_cls._for_instance(obj) return disp diff --git a/lib/sqlalchemy/event/legacy.py b/lib/sqlalchemy/event/legacy.py index 3b1519cb6..7513c7d4d 100644 --- a/lib/sqlalchemy/event/legacy.py +++ b/lib/sqlalchemy/event/legacy.py @@ -22,8 +22,8 @@ def _legacy_signature(since, argnames, converter=None): return leg -def _wrap_fn_for_legacy(dispatch_descriptor, fn, argspec): - for since, argnames, conv in dispatch_descriptor.legacy_signatures: +def _wrap_fn_for_legacy(dispatch_collection, fn, argspec): + for since, argnames, conv in dispatch_collection.legacy_signatures: if argnames[-1] == "**kw": has_kw = True argnames = argnames[0:-1] @@ -40,7 +40,7 @@ def _wrap_fn_for_legacy(dispatch_descriptor, fn, argspec): return fn(*conv(*args)) else: def wrap_leg(*args, **kw): - argdict = dict(zip(dispatch_descriptor.arg_names, args)) + argdict = dict(zip(dispatch_collection.arg_names, args)) args = [argdict[name] for name in argnames] if has_kw: return fn(*args, **kw) @@ -58,16 +58,16 @@ def _indent(text, indent): ) -def _standard_listen_example(dispatch_descriptor, sample_target, fn): +def _standard_listen_example(dispatch_collection, sample_target, fn): example_kw_arg = _indent( "\n".join( "%(arg)s = kw['%(arg)s']" % {"arg": arg} - for arg in dispatch_descriptor.arg_names[0:2] + for arg in dispatch_collection.arg_names[0:2] ), " ") - if dispatch_descriptor.legacy_signatures: + if dispatch_collection.legacy_signatures: current_since = max(since for since, args, conv - in dispatch_descriptor.legacy_signatures) + in dispatch_collection.legacy_signatures) else: current_since = None text = ( @@ -80,7 +80,7 @@ def _standard_listen_example(dispatch_descriptor, sample_target, fn): "\n # ... (event handling logic) ...\n" ) - if len(dispatch_descriptor.arg_names) > 3: + if len(dispatch_collection.arg_names) > 3: text += ( "\n# named argument style (new in 0.9)\n" @@ -96,17 +96,17 @@ def _standard_listen_example(dispatch_descriptor, sample_target, fn): "current_since": " (arguments as of %s)" % current_since if current_since else "", "event_name": fn.__name__, - "has_kw_arguments": ", **kw" if dispatch_descriptor.has_kw else "", - "named_event_arguments": ", ".join(dispatch_descriptor.arg_names), + "has_kw_arguments": ", **kw" if dispatch_collection.has_kw else "", + "named_event_arguments": ", ".join(dispatch_collection.arg_names), "example_kw_arg": example_kw_arg, "sample_target": sample_target } return text -def _legacy_listen_examples(dispatch_descriptor, sample_target, fn): +def _legacy_listen_examples(dispatch_collection, sample_target, fn): text = "" - for since, args, conv in dispatch_descriptor.legacy_signatures: + for since, args, conv in dispatch_collection.legacy_signatures: text += ( "\n# legacy calling style (pre-%(since)s)\n" "@event.listens_for(%(sample_target)s, '%(event_name)s')\n" @@ -117,7 +117,7 @@ def _legacy_listen_examples(dispatch_descriptor, sample_target, fn): "since": since, "event_name": fn.__name__, "has_kw_arguments": " **kw" - if dispatch_descriptor.has_kw else "", + if dispatch_collection.has_kw else "", "named_event_arguments": ", ".join(args), "sample_target": sample_target } @@ -125,8 +125,8 @@ def _legacy_listen_examples(dispatch_descriptor, sample_target, fn): return text -def _version_signature_changes(dispatch_descriptor): - since, args, conv = dispatch_descriptor.legacy_signatures[0] +def _version_signature_changes(dispatch_collection): + since, args, conv = dispatch_collection.legacy_signatures[0] return ( "\n.. versionchanged:: %(since)s\n" " The ``%(event_name)s`` event now accepts the \n" @@ -135,14 +135,14 @@ def _version_signature_changes(dispatch_descriptor): " signature(s) listed above will be automatically \n" " adapted to the new signature." % { "since": since, - "event_name": dispatch_descriptor.__name__, - "named_event_arguments": ", ".join(dispatch_descriptor.arg_names), - "has_kw_arguments": ", **kw" if dispatch_descriptor.has_kw else "" + "event_name": dispatch_collection.name, + "named_event_arguments": ", ".join(dispatch_collection.arg_names), + "has_kw_arguments": ", **kw" if dispatch_collection.has_kw else "" } ) -def _augment_fn_docs(dispatch_descriptor, parent_dispatch_cls, fn): +def _augment_fn_docs(dispatch_collection, parent_dispatch_cls, fn): header = ".. container:: event_signatures\n\n"\ " Example argument forms::\n"\ "\n" @@ -152,16 +152,16 @@ def _augment_fn_docs(dispatch_descriptor, parent_dispatch_cls, fn): header + _indent( _standard_listen_example( - dispatch_descriptor, sample_target, fn), + dispatch_collection, sample_target, fn), " " * 8) ) - if dispatch_descriptor.legacy_signatures: + if dispatch_collection.legacy_signatures: text += _indent( _legacy_listen_examples( - dispatch_descriptor, sample_target, fn), + dispatch_collection, sample_target, fn), " " * 8) - text += _version_signature_changes(dispatch_descriptor) + text += _version_signature_changes(dispatch_collection) return util.inject_docstring_text(fn.__doc__, text, diff --git a/lib/sqlalchemy/event/registry.py b/lib/sqlalchemy/event/registry.py index fc26f91d7..ebc0e6d18 100644 --- a/lib/sqlalchemy/event/registry.py +++ b/lib/sqlalchemy/event/registry.py @@ -37,7 +37,7 @@ listener collections and the listener fn contained _collection_to_key = collections.defaultdict(dict) """ -Given a _ListenerCollection or _DispatchDescriptor, can locate +Given a _ListenerCollection or _ClsLevelListener, can locate all the original listen() arguments and the listener fn contained ref(listenercollection) -> { @@ -191,9 +191,9 @@ class _EventKey(object): target, identifier, fn = \ self.dispatch_target, self.identifier, self._listen_fn - dispatch_descriptor = getattr(target.dispatch, identifier) + dispatch_collection = getattr(target.dispatch, identifier) - adjusted_fn = dispatch_descriptor._adjust_fn_spec(fn, named) + adjusted_fn = dispatch_collection._adjust_fn_spec(fn, named) self = self.with_wrapper(adjusted_fn) @@ -230,13 +230,13 @@ class _EventKey(object): target, identifier, fn = \ self.dispatch_target, self.identifier, self._listen_fn - dispatch_descriptor = getattr(target.dispatch, identifier) + dispatch_collection = getattr(target.dispatch, identifier) if insert: - dispatch_descriptor.\ + dispatch_collection.\ for_modify(target.dispatch).insert(self, propagate) else: - dispatch_descriptor.\ + dispatch_collection.\ for_modify(target.dispatch).append(self, propagate) @property diff --git a/lib/sqlalchemy/orm/events.py b/lib/sqlalchemy/orm/events.py index 9ea0dd834..4d888a350 100644 --- a/lib/sqlalchemy/orm/events.py +++ b/lib/sqlalchemy/orm/events.py @@ -1479,8 +1479,9 @@ class AttributeEvents(event.Events): @staticmethod def _set_dispatch(cls, dispatch_cls): - event.Events._set_dispatch(cls, dispatch_cls) + dispatch = event.Events._set_dispatch(cls, dispatch_cls) dispatch_cls._active_history = False + return dispatch @classmethod def _accept_with(cls, target): diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index c61d93230..9fe6b77f0 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -2643,7 +2643,7 @@ def configure_mappers(): if not Mapper._new_mappers: return - Mapper.dispatch(Mapper).before_configured() + Mapper.dispatch._for_class(Mapper).before_configured() # initialize properties on all mappers # note that _mapper_registry is unordered, which # may randomly conceal/reveal issues related to @@ -2675,7 +2675,7 @@ def configure_mappers(): _already_compiling = False finally: _CONFIGURE_MUTEX.release() - Mapper.dispatch(Mapper).after_configured() + Mapper.dispatch._for_class(Mapper).after_configured() def reconstructor(fn): diff --git a/lib/sqlalchemy/pool.py b/lib/sqlalchemy/pool.py index a147685d9..253bd77b8 100644 --- a/lib/sqlalchemy/pool.py +++ b/lib/sqlalchemy/pool.py @@ -230,6 +230,7 @@ class Pool(log.Identified): % reset_on_return) self.echo = echo + if _dispatch: self.dispatch._update(_dispatch, only_propagate=False) if _dialect: diff --git a/test/base/test_events.py b/test/base/test_events.py index 89379961e..1449bfab0 100644 --- a/test/base/test_events.py +++ b/test/base/test_events.py @@ -154,25 +154,16 @@ class EventsTest(fixtures.TestBase): t2 = self.Target() t1.dispatch.event_one(5, 6) t2.dispatch.event_one(5, 6) - is_( - t1.dispatch.__dict__['event_one'], - self.Target.dispatch.event_one. - _empty_listeners[self.Target] - ) + assert t1.dispatch.event_one in \ + self.Target.dispatch._empty_listeners[self.Target] @event.listens_for(t1, "event_one") def listen_two(x, y): pass - is_not_( - t1.dispatch.__dict__['event_one'], - self.Target.dispatch.event_one. - _empty_listeners[self.Target] - ) - is_( - t2.dispatch.__dict__['event_one'], - self.Target.dispatch.event_one. - _empty_listeners[self.Target] - ) + assert t1.dispatch.event_one not in \ + self.Target.dispatch._empty_listeners[self.Target] + assert t2.dispatch.event_one in \ + self.Target.dispatch._empty_listeners[self.Target] def test_immutable_methods(self): t1 = self.Target() -- cgit v1.2.1 From 27816e8f61b861c5f8718bfb0f1be745ef204040 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 18:30:25 -0500 Subject: - strategies + declarative --- lib/sqlalchemy/ext/declarative/clsregistry.py | 10 ++++++++++ lib/sqlalchemy/orm/interfaces.py | 2 ++ lib/sqlalchemy/orm/strategies.py | 23 +++++++++++++++++++++++ lib/sqlalchemy/sql/visitors.py | 1 + 4 files changed, 36 insertions(+) diff --git a/lib/sqlalchemy/ext/declarative/clsregistry.py b/lib/sqlalchemy/ext/declarative/clsregistry.py index 3ef63a5ae..d2a09d823 100644 --- a/lib/sqlalchemy/ext/declarative/clsregistry.py +++ b/lib/sqlalchemy/ext/declarative/clsregistry.py @@ -71,6 +71,8 @@ class _MultipleClassMarker(object): """ + __slots__ = 'on_remove', 'contents', '__weakref__' + def __init__(self, classes, on_remove=None): self.on_remove = on_remove self.contents = set([ @@ -127,6 +129,8 @@ class _ModuleMarker(object): """ + __slots__ = 'parent', 'name', 'contents', 'mod_ns', 'path', '__weakref__' + def __init__(self, name, parent): self.parent = parent self.name = name @@ -172,6 +176,8 @@ class _ModuleMarker(object): class _ModNS(object): + __slots__ = '__parent', + def __init__(self, parent): self.__parent = parent @@ -193,6 +199,8 @@ class _ModNS(object): class _GetColumns(object): + __slots__ = 'cls', + def __init__(self, cls): self.cls = cls @@ -221,6 +229,8 @@ inspection._inspects(_GetColumns)( class _GetTable(object): + __slots__ = 'key', 'metadata' + def __init__(self, key, metadata): self.key = key self.metadata = metadata diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index bff73258c..68b86268c 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -560,6 +560,8 @@ class LoaderStrategy(object): """ + __slots__ = 'parent_property', 'is_class_level', 'parent', 'key' + def __init__(self, parent): self.parent_property = parent self.is_class_level = False diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index d95f17f64..4e47a8f24 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -105,6 +105,8 @@ class UninstrumentedColumnLoader(LoaderStrategy): if the argument is against the with_polymorphic selectable. """ + __slots__ = 'columns', + def __init__(self, parent): super(UninstrumentedColumnLoader, self).__init__(parent) self.columns = self.parent_property.columns @@ -128,6 +130,8 @@ class UninstrumentedColumnLoader(LoaderStrategy): class ColumnLoader(LoaderStrategy): """Provide loading behavior for a :class:`.ColumnProperty`.""" + __slots__ = 'columns', 'is_composite' + def __init__(self, parent): super(ColumnLoader, self).__init__(parent) self.columns = self.parent_property.columns @@ -176,6 +180,8 @@ class ColumnLoader(LoaderStrategy): class DeferredColumnLoader(LoaderStrategy): """Provide loading behavior for a deferred :class:`.ColumnProperty`.""" + __slots__ = 'columns', 'group' + def __init__(self, parent): super(DeferredColumnLoader, self).__init__(parent) if hasattr(self.parent_property, 'composite_class'): @@ -300,6 +306,8 @@ class LoadDeferredColumns(object): class AbstractRelationshipLoader(LoaderStrategy): """LoaderStratgies which deal with related objects.""" + __slots__ = 'mapper', 'target', 'uselist' + def __init__(self, parent): super(AbstractRelationshipLoader, self).__init__(parent) self.mapper = self.parent_property.mapper @@ -316,6 +324,8 @@ class NoLoader(AbstractRelationshipLoader): """ + __slots__ = () + def init_class_attribute(self, mapper): self.is_class_level = True @@ -343,6 +353,10 @@ class LazyLoader(AbstractRelationshipLoader): """ + __slots__ = ( + '_lazywhere', '_rev_lazywhere', 'use_get', '_bind_to_col', + '_equated_columns', '_rev_bind_to_col', '_rev_equated_columns') + def __init__(self, parent): super(LazyLoader, self).__init__(parent) join_condition = self.parent_property._join_condition @@ -647,6 +661,8 @@ class LazyLoader(AbstractRelationshipLoader): class LoadLazyAttribute(object): """serializable loader object used by LazyLoader""" + __slots__ = 'key', + def __init__(self, key): self.key = key @@ -661,6 +677,8 @@ class LoadLazyAttribute(object): @properties.RelationshipProperty.strategy_for(lazy="immediate") class ImmediateLoader(AbstractRelationshipLoader): + __slots__ = () + def init_class_attribute(self, mapper): self.parent_property.\ _get_strategy_by_cls(LazyLoader).\ @@ -684,6 +702,8 @@ class ImmediateLoader(AbstractRelationshipLoader): @log.class_logger @properties.RelationshipProperty.strategy_for(lazy="subquery") class SubqueryLoader(AbstractRelationshipLoader): + __slots__ = 'join_depth', + def __init__(self, parent): super(SubqueryLoader, self).__init__(parent) self.join_depth = self.parent_property.join_depth @@ -1069,6 +1089,9 @@ class JoinedLoader(AbstractRelationshipLoader): using joined eager loading. """ + + __slots__ = 'join_depth', + def __init__(self, parent): super(JoinedLoader, self).__init__(parent) self.join_depth = self.parent_property.join_depth diff --git a/lib/sqlalchemy/sql/visitors.py b/lib/sqlalchemy/sql/visitors.py index bb525744a..d09b82148 100644 --- a/lib/sqlalchemy/sql/visitors.py +++ b/lib/sqlalchemy/sql/visitors.py @@ -51,6 +51,7 @@ class VisitableType(type): Classes having no __visit_name__ attribute will remain unaffected. """ + def __init__(cls, clsname, bases, clsdict): if clsname != 'Visitable' and \ hasattr(cls, '__visit_name__'): -- cgit v1.2.1 From 46a35b7647ac9213a2d968e74b50ca070f30abe2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 18:45:30 -0500 Subject: - clean up default comparator which doesn't need to be a class, get PG stuff working --- lib/sqlalchemy/dialects/postgresql/base.py | 11 +- lib/sqlalchemy/dialects/postgresql/json.py | 5 +- lib/sqlalchemy/sql/default_comparator.py | 511 +++++++++++++++-------------- lib/sqlalchemy/sql/type_api.py | 12 +- 4 files changed, 272 insertions(+), 267 deletions(-) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index d870dd295..fa9a2cfd0 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -483,7 +483,7 @@ import re from ... import sql, schema, exc, util from ...engine import default, reflection -from ...sql import compiler, expression, operators +from ...sql import compiler, expression, operators, default_comparator from ... import types as sqltypes try: @@ -680,10 +680,10 @@ class _Slice(expression.ColumnElement): type = sqltypes.NULLTYPE def __init__(self, slice_, source_comparator): - self.start = source_comparator._check_literal( + self.start = default_comparator._check_literal( source_comparator.expr, operators.getitem, slice_.start) - self.stop = source_comparator._check_literal( + self.stop = default_comparator._check_literal( source_comparator.expr, operators.getitem, slice_.stop) @@ -876,8 +876,9 @@ class ARRAY(sqltypes.Concatenable, sqltypes.TypeEngine): index += shift_indexes return_type = self.type.item_type - return self._binary_operate(self.expr, operators.getitem, index, - result_type=return_type) + return default_comparator._binary_operate( + self.expr, operators.getitem, index, + result_type=return_type) def any(self, other, operator=operators.eq): """Return ``other operator ANY (array)`` clause. diff --git a/lib/sqlalchemy/dialects/postgresql/json.py b/lib/sqlalchemy/dialects/postgresql/json.py index 50176918e..f38c4a56a 100644 --- a/lib/sqlalchemy/dialects/postgresql/json.py +++ b/lib/sqlalchemy/dialects/postgresql/json.py @@ -12,7 +12,7 @@ from .base import ischema_names from ... import types as sqltypes from ...sql.operators import custom_op from ... import sql -from ...sql import elements +from ...sql import elements, default_comparator from ... import util __all__ = ('JSON', 'JSONElement', 'JSONB') @@ -46,7 +46,8 @@ class JSONElement(elements.BinaryExpression): self._json_opstring = opstring operator = custom_op(opstring, precedence=5) - right = left._check_literal(left, operator, right) + right = default_comparator._check_literal( + left, operator, right) super(JSONElement, self).__init__( left, right, operator, type_=result_type) diff --git a/lib/sqlalchemy/sql/default_comparator.py b/lib/sqlalchemy/sql/default_comparator.py index c898b78d6..bb9e53aae 100644 --- a/lib/sqlalchemy/sql/default_comparator.py +++ b/lib/sqlalchemy/sql/default_comparator.py @@ -18,267 +18,270 @@ from .elements import BindParameter, True_, False_, BinaryExpression, \ from .selectable import SelectBase, Alias, Selectable, ScalarSelect -class _DefaultColumnComparator(object): - """Defines comparison and math operations. - - See :class:`.ColumnOperators` and :class:`.Operators` for descriptions - of all operations. - - """ - - def _boolean_compare(self, expr, op, obj, negate=None, reverse=False, - _python_is_types=(util.NoneType, bool), - result_type = None, - **kwargs): - - if result_type is None: - result_type = type_api.BOOLEANTYPE - - if isinstance(obj, _python_is_types + (Null, True_, False_)): - - # allow x ==/!= True/False to be treated as a literal. - # this comes out to "== / != true/false" or "1/0" if those - # constants aren't supported and works on all platforms - if op in (operators.eq, operators.ne) and \ - isinstance(obj, (bool, True_, False_)): - return BinaryExpression(expr, - _literal_as_text(obj), - op, - type_=result_type, - negate=negate, modifiers=kwargs) - else: - # all other None/True/False uses IS, IS NOT - if op in (operators.eq, operators.is_): - return BinaryExpression(expr, _const_expr(obj), - operators.is_, - negate=operators.isnot) - elif op in (operators.ne, operators.isnot): - return BinaryExpression(expr, _const_expr(obj), - operators.isnot, - negate=operators.is_) - else: - raise exc.ArgumentError( - "Only '=', '!=', 'is_()', 'isnot()' operators can " - "be used with None/True/False") - else: - obj = self._check_literal(expr, op, obj) +def _boolean_compare(expr, op, obj, negate=None, reverse=False, + _python_is_types=(util.NoneType, bool), + result_type = None, + **kwargs): - if reverse: - return BinaryExpression(obj, - expr, - op, - type_=result_type, - negate=negate, modifiers=kwargs) - else: + if result_type is None: + result_type = type_api.BOOLEANTYPE + + if isinstance(obj, _python_is_types + (Null, True_, False_)): + + # allow x ==/!= True/False to be treated as a literal. + # this comes out to "== / != true/false" or "1/0" if those + # constants aren't supported and works on all platforms + if op in (operators.eq, operators.ne) and \ + isinstance(obj, (bool, True_, False_)): return BinaryExpression(expr, - obj, + _literal_as_text(obj), op, type_=result_type, negate=negate, modifiers=kwargs) - - def _binary_operate(self, expr, op, obj, reverse=False, result_type=None, - **kw): - obj = self._check_literal(expr, op, obj) - - if reverse: - left, right = obj, expr - else: - left, right = expr, obj - - if result_type is None: - op, result_type = left.comparator._adapt_expression( - op, right.comparator) - - return BinaryExpression( - left, right, op, type_=result_type, modifiers=kw) - - def _conjunction_operate(self, expr, op, other, **kw): - if op is operators.and_: - return and_(expr, other) - elif op is operators.or_: - return or_(expr, other) else: - raise NotImplementedError() - - def _scalar(self, expr, op, fn, **kw): - return fn(expr) - - def _in_impl(self, expr, op, seq_or_selectable, negate_op, **kw): - seq_or_selectable = _clause_element_as_expr(seq_or_selectable) - - if isinstance(seq_or_selectable, ScalarSelect): - return self._boolean_compare(expr, op, seq_or_selectable, - negate=negate_op) - elif isinstance(seq_or_selectable, SelectBase): - - # TODO: if we ever want to support (x, y, z) IN (select x, - # y, z from table), we would need a multi-column version of - # as_scalar() to produce a multi- column selectable that - # does not export itself as a FROM clause - - return self._boolean_compare( - expr, op, seq_or_selectable.as_scalar(), - negate=negate_op, **kw) - elif isinstance(seq_or_selectable, (Selectable, TextClause)): - return self._boolean_compare(expr, op, seq_or_selectable, - negate=negate_op, **kw) - elif isinstance(seq_or_selectable, ClauseElement): - raise exc.InvalidRequestError( - 'in_() accepts' - ' either a list of expressions ' - 'or a selectable: %r' % seq_or_selectable) - - # Handle non selectable arguments as sequences - args = [] - for o in seq_or_selectable: - if not _is_literal(o): - if not isinstance(o, operators.ColumnOperators): - raise exc.InvalidRequestError( - 'in_() accepts' - ' either a list of expressions ' - 'or a selectable: %r' % o) - elif o is None: - o = Null() + # all other None/True/False uses IS, IS NOT + if op in (operators.eq, operators.is_): + return BinaryExpression(expr, _const_expr(obj), + operators.is_, + negate=operators.isnot) + elif op in (operators.ne, operators.isnot): + return BinaryExpression(expr, _const_expr(obj), + operators.isnot, + negate=operators.is_) else: - o = expr._bind_param(op, o) - args.append(o) - if len(args) == 0: - - # Special case handling for empty IN's, behave like - # comparison against zero row selectable. We use != to - # build the contradiction as it handles NULL values - # appropriately, i.e. "not (x IN ())" should not return NULL - # values for x. - - util.warn('The IN-predicate on "%s" was invoked with an ' - 'empty sequence. This results in a ' - 'contradiction, which nonetheless can be ' - 'expensive to evaluate. Consider alternative ' - 'strategies for improved performance.' % expr) - if op is operators.in_op: - return expr != expr - else: - return expr == expr - - return self._boolean_compare(expr, op, - ClauseList(*args).self_group(against=op), - negate=negate_op) - - def _unsupported_impl(self, expr, op, *arg, **kw): - raise NotImplementedError("Operator '%s' is not supported on " - "this expression" % op.__name__) - - def _inv_impl(self, expr, op, **kw): - """See :meth:`.ColumnOperators.__inv__`.""" - if hasattr(expr, 'negation_clause'): - return expr.negation_clause + raise exc.ArgumentError( + "Only '=', '!=', 'is_()', 'isnot()' operators can " + "be used with None/True/False") + else: + obj = _check_literal(expr, op, obj) + + if reverse: + return BinaryExpression(obj, + expr, + op, + type_=result_type, + negate=negate, modifiers=kwargs) + else: + return BinaryExpression(expr, + obj, + op, + type_=result_type, + negate=negate, modifiers=kwargs) + + +def _binary_operate(expr, op, obj, reverse=False, result_type=None, + **kw): + obj = _check_literal(expr, op, obj) + + if reverse: + left, right = obj, expr + else: + left, right = expr, obj + + if result_type is None: + op, result_type = left.comparator._adapt_expression( + op, right.comparator) + + return BinaryExpression( + left, right, op, type_=result_type, modifiers=kw) + + +def _conjunction_operate(expr, op, other, **kw): + if op is operators.and_: + return and_(expr, other) + elif op is operators.or_: + return or_(expr, other) + else: + raise NotImplementedError() + + +def _scalar(expr, op, fn, **kw): + return fn(expr) + + +def _in_impl(expr, op, seq_or_selectable, negate_op, **kw): + seq_or_selectable = _clause_element_as_expr(seq_or_selectable) + + if isinstance(seq_or_selectable, ScalarSelect): + return _boolean_compare(expr, op, seq_or_selectable, + negate=negate_op) + elif isinstance(seq_or_selectable, SelectBase): + + # TODO: if we ever want to support (x, y, z) IN (select x, + # y, z from table), we would need a multi-column version of + # as_scalar() to produce a multi- column selectable that + # does not export itself as a FROM clause + + return _boolean_compare( + expr, op, seq_or_selectable.as_scalar(), + negate=negate_op, **kw) + elif isinstance(seq_or_selectable, (Selectable, TextClause)): + return _boolean_compare(expr, op, seq_or_selectable, + negate=negate_op, **kw) + elif isinstance(seq_or_selectable, ClauseElement): + raise exc.InvalidRequestError( + 'in_() accepts' + ' either a list of expressions ' + 'or a selectable: %r' % seq_or_selectable) + + # Handle non selectable arguments as sequences + args = [] + for o in seq_or_selectable: + if not _is_literal(o): + if not isinstance(o, operators.ColumnOperators): + raise exc.InvalidRequestError( + 'in_() accepts' + ' either a list of expressions ' + 'or a selectable: %r' % o) + elif o is None: + o = Null() else: - return expr._negate() - - def _neg_impl(self, expr, op, **kw): - """See :meth:`.ColumnOperators.__neg__`.""" - return UnaryExpression(expr, operator=operators.neg) - - def _match_impl(self, expr, op, other, **kw): - """See :meth:`.ColumnOperators.match`.""" - - return self._boolean_compare( - expr, operators.match_op, - self._check_literal( - expr, operators.match_op, other), - result_type=type_api.MATCHTYPE, - negate=operators.notmatch_op - if op is operators.match_op else operators.match_op, - **kw - ) - - def _distinct_impl(self, expr, op, **kw): - """See :meth:`.ColumnOperators.distinct`.""" - return UnaryExpression(expr, operator=operators.distinct_op, - type_=expr.type) - - def _between_impl(self, expr, op, cleft, cright, **kw): - """See :meth:`.ColumnOperators.between`.""" - return BinaryExpression( - expr, - ClauseList( - self._check_literal(expr, operators.and_, cleft), - self._check_literal(expr, operators.and_, cright), - operator=operators.and_, - group=False, group_contents=False), - op, - negate=operators.notbetween_op - if op is operators.between_op - else operators.between_op, - modifiers=kw) - - def _collate_impl(self, expr, op, other, **kw): - return collate(expr, other) - - # a mapping of operators with the method they use, along with - # their negated operator for comparison operators - operators = { - "and_": (_conjunction_operate,), - "or_": (_conjunction_operate,), - "inv": (_inv_impl,), - "add": (_binary_operate,), - "mul": (_binary_operate,), - "sub": (_binary_operate,), - "div": (_binary_operate,), - "mod": (_binary_operate,), - "truediv": (_binary_operate,), - "custom_op": (_binary_operate,), - "concat_op": (_binary_operate,), - "lt": (_boolean_compare, operators.ge), - "le": (_boolean_compare, operators.gt), - "ne": (_boolean_compare, operators.eq), - "gt": (_boolean_compare, operators.le), - "ge": (_boolean_compare, operators.lt), - "eq": (_boolean_compare, operators.ne), - "like_op": (_boolean_compare, operators.notlike_op), - "ilike_op": (_boolean_compare, operators.notilike_op), - "notlike_op": (_boolean_compare, operators.like_op), - "notilike_op": (_boolean_compare, operators.ilike_op), - "contains_op": (_boolean_compare, operators.notcontains_op), - "startswith_op": (_boolean_compare, operators.notstartswith_op), - "endswith_op": (_boolean_compare, operators.notendswith_op), - "desc_op": (_scalar, UnaryExpression._create_desc), - "asc_op": (_scalar, UnaryExpression._create_asc), - "nullsfirst_op": (_scalar, UnaryExpression._create_nullsfirst), - "nullslast_op": (_scalar, UnaryExpression._create_nullslast), - "in_op": (_in_impl, operators.notin_op), - "notin_op": (_in_impl, operators.in_op), - "is_": (_boolean_compare, operators.is_), - "isnot": (_boolean_compare, operators.isnot), - "collate": (_collate_impl,), - "match_op": (_match_impl,), - "notmatch_op": (_match_impl,), - "distinct_op": (_distinct_impl,), - "between_op": (_between_impl, ), - "notbetween_op": (_between_impl, ), - "neg": (_neg_impl,), - "getitem": (_unsupported_impl,), - "lshift": (_unsupported_impl,), - "rshift": (_unsupported_impl,), - } - - def _check_literal(self, expr, operator, other): - if isinstance(other, (ColumnElement, TextClause)): - if isinstance(other, BindParameter) and \ - other.type._isnull: - other = other._clone() - other.type = expr.type - return other - elif hasattr(other, '__clause_element__'): - other = other.__clause_element__() - elif isinstance(other, type_api.TypeEngine.Comparator): - other = other.expr - - if isinstance(other, (SelectBase, Alias)): - return other.as_scalar() - elif not isinstance(other, (ColumnElement, TextClause)): - return expr._bind_param(operator, other) + o = expr._bind_param(op, o) + args.append(o) + if len(args) == 0: + + # Special case handling for empty IN's, behave like + # comparison against zero row selectable. We use != to + # build the contradiction as it handles NULL values + # appropriately, i.e. "not (x IN ())" should not return NULL + # values for x. + + util.warn('The IN-predicate on "%s" was invoked with an ' + 'empty sequence. This results in a ' + 'contradiction, which nonetheless can be ' + 'expensive to evaluate. Consider alternative ' + 'strategies for improved performance.' % expr) + if op is operators.in_op: + return expr != expr else: - return other + return expr == expr + + return _boolean_compare(expr, op, + ClauseList(*args).self_group(against=op), + negate=negate_op) + + +def _unsupported_impl(expr, op, *arg, **kw): + raise NotImplementedError("Operator '%s' is not supported on " + "this expression" % op.__name__) + + +def _inv_impl(expr, op, **kw): + """See :meth:`.ColumnOperators.__inv__`.""" + if hasattr(expr, 'negation_clause'): + return expr.negation_clause + else: + return expr._negate() + + +def _neg_impl(expr, op, **kw): + """See :meth:`.ColumnOperators.__neg__`.""" + return UnaryExpression(expr, operator=operators.neg) + + +def _match_impl(expr, op, other, **kw): + """See :meth:`.ColumnOperators.match`.""" + + return _boolean_compare( + expr, operators.match_op, + _check_literal( + expr, operators.match_op, other), + result_type=type_api.MATCHTYPE, + negate=operators.notmatch_op + if op is operators.match_op else operators.match_op, + **kw + ) + + +def _distinct_impl(expr, op, **kw): + """See :meth:`.ColumnOperators.distinct`.""" + return UnaryExpression(expr, operator=operators.distinct_op, + type_=expr.type) + + +def _between_impl(expr, op, cleft, cright, **kw): + """See :meth:`.ColumnOperators.between`.""" + return BinaryExpression( + expr, + ClauseList( + _check_literal(expr, operators.and_, cleft), + _check_literal(expr, operators.and_, cright), + operator=operators.and_, + group=False, group_contents=False), + op, + negate=operators.notbetween_op + if op is operators.between_op + else operators.between_op, + modifiers=kw) + + +def _collate_impl(expr, op, other, **kw): + return collate(expr, other) + +# a mapping of operators with the method they use, along with +# their negated operator for comparison operators +operator_lookup = { + "and_": (_conjunction_operate,), + "or_": (_conjunction_operate,), + "inv": (_inv_impl,), + "add": (_binary_operate,), + "mul": (_binary_operate,), + "sub": (_binary_operate,), + "div": (_binary_operate,), + "mod": (_binary_operate,), + "truediv": (_binary_operate,), + "custom_op": (_binary_operate,), + "concat_op": (_binary_operate,), + "lt": (_boolean_compare, operators.ge), + "le": (_boolean_compare, operators.gt), + "ne": (_boolean_compare, operators.eq), + "gt": (_boolean_compare, operators.le), + "ge": (_boolean_compare, operators.lt), + "eq": (_boolean_compare, operators.ne), + "like_op": (_boolean_compare, operators.notlike_op), + "ilike_op": (_boolean_compare, operators.notilike_op), + "notlike_op": (_boolean_compare, operators.like_op), + "notilike_op": (_boolean_compare, operators.ilike_op), + "contains_op": (_boolean_compare, operators.notcontains_op), + "startswith_op": (_boolean_compare, operators.notstartswith_op), + "endswith_op": (_boolean_compare, operators.notendswith_op), + "desc_op": (_scalar, UnaryExpression._create_desc), + "asc_op": (_scalar, UnaryExpression._create_asc), + "nullsfirst_op": (_scalar, UnaryExpression._create_nullsfirst), + "nullslast_op": (_scalar, UnaryExpression._create_nullslast), + "in_op": (_in_impl, operators.notin_op), + "notin_op": (_in_impl, operators.in_op), + "is_": (_boolean_compare, operators.is_), + "isnot": (_boolean_compare, operators.isnot), + "collate": (_collate_impl,), + "match_op": (_match_impl,), + "notmatch_op": (_match_impl,), + "distinct_op": (_distinct_impl,), + "between_op": (_between_impl, ), + "notbetween_op": (_between_impl, ), + "neg": (_neg_impl,), + "getitem": (_unsupported_impl,), + "lshift": (_unsupported_impl,), + "rshift": (_unsupported_impl,), +} + + +def _check_literal(expr, operator, other): + if isinstance(other, (ColumnElement, TextClause)): + if isinstance(other, BindParameter) and \ + other.type._isnull: + other = other._clone() + other.type = expr.type + return other + elif hasattr(other, '__clause_element__'): + other = other.__clause_element__() + elif isinstance(other, type_api.TypeEngine.Comparator): + other = other.expr + + if isinstance(other, (SelectBase, Alias)): + return other.as_scalar() + elif not isinstance(other, (ColumnElement, TextClause)): + return expr._bind_param(operator, other) + else: + return other -the_comparator = _DefaultColumnComparator() diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py index 834640928..bff497800 100644 --- a/lib/sqlalchemy/sql/type_api.py +++ b/lib/sqlalchemy/sql/type_api.py @@ -48,21 +48,21 @@ class TypeEngine(Visitable): """ __slots__ = 'expr', 'type' + default_comparator = None + def __init__(self, expr): self.expr = expr self.type = expr.type @util.dependencies('sqlalchemy.sql.default_comparator') def operate(self, default_comparator, op, *other, **kwargs): - comp = default_comparator.the_comparator - o = comp.operators[op.__name__] - return o[0](comp, self.expr, op, *(other + o[1:]), **kwargs) + o = default_comparator.operator_lookup[op.__name__] + return o[0](self.expr, op, *(other + o[1:]), **kwargs) @util.dependencies('sqlalchemy.sql.default_comparator') def reverse_operate(self, default_comparator, op, other, **kwargs): - comp = default_comparator.the_comparator - o = comp.operators[op.__name__] - return o[0](comp, self.expr, op, other, + o = default_comparator.operator_lookup[op.__name__] + return o[0](self.expr, op, other, reverse=True, *o[1:], **kwargs) def _adapt_expression(self, op, other_comparator): -- cgit v1.2.1 From b3bf7915b59c9c749d335984e03b386605516d0f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 19:05:31 -0500 Subject: - scale back _Dispatch and _JoinedDispatcher to use a __getitem__ scheme to start up listener collections; this pulls the overhead off of construction and makes performance much like the descriptor version, while still allowing slots. Fix up some profiles. --- lib/sqlalchemy/event/base.py | 41 +++++++++++++++++++++++++++++------------ test/base/test_events.py | 18 ++++++++++++------ test/profiles.txt | 18 +++++++++--------- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/lib/sqlalchemy/event/base.py b/lib/sqlalchemy/event/base.py index 962d850c2..2d5468886 100644 --- a/lib/sqlalchemy/event/base.py +++ b/lib/sqlalchemy/event/base.py @@ -67,23 +67,35 @@ class _Dispatch(object): # in one ORM edge case, an attribute is added to _Dispatch, # so __dict__ is used in just that case and potentially others. - __slots__ = '_parent', '_instance_cls', '__dict__' + __slots__ = '_parent', '_instance_cls', '__dict__', '_empty_listeners' - _empty_listeners = weakref.WeakKeyDictionary() + _empty_listener_reg = weakref.WeakKeyDictionary() def __init__(self, parent, instance_cls=None): self._parent = parent self._instance_cls = instance_cls if instance_cls: try: - _empty_listeners = self._empty_listeners[instance_cls] + self._empty_listeners = self._empty_listener_reg[instance_cls] except KeyError: - _empty_listeners = self._empty_listeners[instance_cls] = [ - _EmptyListener(ls, instance_cls) - for ls in parent._event_descriptors - ] - for ls in _empty_listeners: - setattr(self, ls.name, ls) + self._empty_listeners = \ + self._empty_listener_reg[instance_cls] = dict( + (ls.name, _EmptyListener(ls, instance_cls)) + for ls in parent._event_descriptors + ) + else: + self._empty_listeners = {} + + def __getattr__(self, name): + # assign EmptyListeners as attributes on demand + # to reduce startup time for new dispatch objects + try: + ls = self._empty_listeners[name] + except KeyError: + raise AttributeError(name) + else: + setattr(self, ls.name, ls) + return ls @property def _event_descriptors(self): @@ -244,9 +256,14 @@ class _JoinedDispatcher(object): self.local = local self.parent = parent self._instance_cls = self.local._instance_cls - for ls in local._event_descriptors: - setattr(self, ls.name, _JoinedListener( - parent, ls.name, ls)) + + def __getattr__(self, name): + # assign _JoinedListeners as attributes on demand + # to reduce startup time for new dispatch objects + ls = getattr(self.local, name) + jl = _JoinedListener(self.parent, ls.name, ls) + setattr(self, ls.name, jl) + return jl @property def _listen(self): diff --git a/test/base/test_events.py b/test/base/test_events.py index 1449bfab0..8cfbd0180 100644 --- a/test/base/test_events.py +++ b/test/base/test_events.py @@ -154,16 +154,22 @@ class EventsTest(fixtures.TestBase): t2 = self.Target() t1.dispatch.event_one(5, 6) t2.dispatch.event_one(5, 6) - assert t1.dispatch.event_one in \ - self.Target.dispatch._empty_listeners[self.Target] + is_( + self.Target.dispatch._empty_listener_reg[self.Target]['event_one'], + t1.dispatch.event_one + ) @event.listens_for(t1, "event_one") def listen_two(x, y): pass - assert t1.dispatch.event_one not in \ - self.Target.dispatch._empty_listeners[self.Target] - assert t2.dispatch.event_one in \ - self.Target.dispatch._empty_listeners[self.Target] + is_not_( + self.Target.dispatch._empty_listener_reg[self.Target]['event_one'], + t1.dispatch.event_one + ) + is_( + self.Target.dispatch._empty_listener_reg[self.Target]['event_one'], + t2.dispatch.event_one + ) def test_immutable_methods(self): t1 = self.Target() diff --git a/test/profiles.txt b/test/profiles.txt index c11000e29..507762e65 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -133,8 +133,8 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycop test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 19280 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 28297 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_nocextensions 29138 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 32398 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 37327 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 20352 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 29355 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_cextensions 20135 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_nocextensions 29138 @@ -147,7 +147,7 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 27144 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 30149 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_nocextensions 29068 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 32197 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 26208 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 31179 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_cextensions 26065 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_nocextensions 29068 @@ -214,11 +214,11 @@ test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_mysql_mysqldb_cexte test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_mysql_mysqldb_nocextensions 117,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_cextensions 117,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_nocextensions 117,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 117,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 117,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 91,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 91,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_nocextensions 122,19 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 122,19 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 122,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 94,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 94,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_cextensions 122,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_nocextensions 122,19 @@ -351,8 +351,8 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite # TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5562,292,3697,11893,1106,1968,2433 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5606,292,3929,13595,1223,2011,2692 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5892,292,3697,11893,1106,1968,2433 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5936,292,3929,13595,1223,2011,2692 test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 5497,274,3609,11647,1097,1921,2486 test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 5519,274,3705,12819,1191,1928,2678 test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_cextensions 5497,273,3577,11529,1077,1883,2439 -- cgit v1.2.1 From 18d3dc716390c56524370146a2635ed3659ec82d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 19:28:08 -0500 Subject: - Structural memory use has been improved via much more significant use of ``__slots__`` for many internal objects. This optimization is particularly geared towards the base memory size of large applications that have lots of tables and columns, and greatly reduces memory size for a variety of high-volume objects including event listening internals, comparator objects and parts of the ORM attribute and loader strategy system. --- doc/build/changelog/changelog_10.rst | 15 +++++++++++++++ doc/build/changelog/migration_10.rst | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index bfe2ebbc6..7f9fbff91 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,21 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, general + + Structural memory use has been improved via much more significant use + of ``__slots__`` for many internal objects. This optimization is + particularly geared towards the base memory size of large applications + that have lots of tables and columns, and greatly reduces memory + size for a variety of high-volume objects including event listening + internals, comparator objects and parts of the ORM attribute and + loader strategy system. + + .. seealso:: + + :ref:`feature_slots` + .. change:: :tags: bug, mysql :tickets: 3283 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 79756ec17..e5382be54 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -8,7 +8,7 @@ What's New in SQLAlchemy 1.0? undergoing maintenance releases as of May, 2014, and SQLAlchemy version 1.0, as of yet unreleased. - Document last updated: December 14, 2014 + Document last updated: January 4, 2015 Introduction ============ @@ -286,6 +286,37 @@ object totally smokes both namedtuple and KeyedTuple:: :ticket:`3176` +.. _feature_slots: + +Significant Improvements in Structural Memory Use +-------------------------------------------------- + +Structural memory use has been improved via much more significant use +of ``__slots__`` for many internal objects. This optimization is +particularly geared towards the base memory size of large applications +that have lots of tables and columns, and reduces memory +size for a variety of high-volume objects including event listening +internals, comparator objects and parts of the ORM attribute and +loader strategy system. + +A bench that makes use of heapy measure the startup size of Nova +illustrates a difference of about 2 megs of memory, a total of 27% +of memory taken up by SQLAlchemy's objects, associated dictionaries, as +well as weakrefs, within a basic import of "nova.db.sqlalchemy.models":: + + # reported by heapy, summation of SQLAlchemy objects + + # associated dicts + weakref-related objects with core of Nova imported: + + Before: total count 26477 total bytes 7975712 + After: total count 21413 total bytes 5752976 + + # reported for the Python module space overall with the + # core of Nova imported: + + Before: Partition of a set of 355558 objects. Total size = 61661760 bytes. + After: Partition of a set of 350281 objects. Total size = 59415104 bytes. + + .. _feature_updatemany: UPDATE statements are now batched with executemany() in a flush -- cgit v1.2.1 From 2f6ec41dc98c83c66e36d85fd236bd97402cadb4 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 19:43:48 -0500 Subject: - fix test for new events --- test/ext/test_extendedattr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ext/test_extendedattr.py b/test/ext/test_extendedattr.py index 352b6b241..c7627c8b2 100644 --- a/test/ext/test_extendedattr.py +++ b/test/ext/test_extendedattr.py @@ -485,5 +485,5 @@ class ExtendedEventsTest(fixtures.ORMTest): register_class(A) manager = instrumentation.manager_of_class(A) - assert issubclass(manager.dispatch._parent_cls.__dict__['dispatch'].events, MyEvents) + assert issubclass(manager.dispatch._events, MyEvents) -- cgit v1.2.1 From 2a28dd93698b345f718e261adc7ba0a27d382058 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 21:01:27 -0500 Subject: - callcounts - this needs to be serializable and isn't high volume so just whack the slots --- lib/sqlalchemy/orm/strategies.py | 2 -- test/profiles.txt | 42 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 4e47a8f24..81da7ba93 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -661,8 +661,6 @@ class LazyLoader(AbstractRelationshipLoader): class LoadLazyAttribute(object): """serializable loader object used by LazyLoader""" - __slots__ = 'key', - def __init__(self, key): self.key = key diff --git a/test/profiles.txt b/test/profiles.txt index 507762e65..b241653e6 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -16,9 +16,9 @@ test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_mysql_mysqldb_cextensions 74 test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_mysql_mysqldb_nocextensions 74 test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_cextensions 74 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_nocextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_nocextensions 76 test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_cextensions 74 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_nocextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_nocextensions 76 test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_postgresql_psycopg2_cextensions 77 test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_postgresql_psycopg2_nocextensions 77 test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_sqlite_pysqlite_cextensions 77 @@ -33,9 +33,9 @@ test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_sqlite_pysqlite_noc test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_mysql_mysqldb_cextensions 152 test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_mysql_mysqldb_nocextensions 152 test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_cextensions 152 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_nocextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_nocextensions 154 test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_cextensions 152 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_nocextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_nocextensions 154 test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_postgresql_psycopg2_cextensions 165 test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_postgresql_psycopg2_nocextensions 165 test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_sqlite_pysqlite_cextensions 165 @@ -50,9 +50,9 @@ test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_sqlite_pysqlite_noc test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_mysql_mysqldb_cextensions 186 test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_mysql_mysqldb_nocextensions 186 test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_cextensions 186 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_nocextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_nocextensions 189 test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_cextensions 186 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_nocextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_nocextensions 189 test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_postgresql_psycopg2_cextensions 199 test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_postgresql_psycopg2_nocextensions 199 test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_sqlite_pysqlite_cextensions 199 @@ -67,7 +67,7 @@ test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_sqlite_pysql test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_mysql_mysqldb_cextensions 79 test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_mysql_mysqldb_nocextensions 79 test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_cextensions 77 -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_nocextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_nocextensions 79 test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_sqlite_pysqlite_cextensions 77 test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_sqlite_pysqlite_nocextensions 77 test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_postgresql_psycopg2_cextensions 78 @@ -103,7 +103,7 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_mysql_m test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgresql_psycopg2_cextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgresql_psycopg2_nocextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_cextensions 4265 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4260 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4262 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_nocextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_nocextensions 4266 @@ -131,7 +131,7 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_mysql_mysqldb_noc test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_cextensions 31132 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_nocextensions 40149 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 19280 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 28297 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 28347 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_nocextensions 29138 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 20352 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 29355 @@ -145,7 +145,7 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_mysql_mysq test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_cextensions 27049 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_nocextensions 30054 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 27144 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 30149 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 28183 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_nocextensions 29068 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 26208 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 31179 @@ -173,7 +173,7 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_postgresql_psycopg2_cextensions 119849 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_postgresql_psycopg2_nocextensions 122553 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_cextensions 162315 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 165111 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 164551 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_nocextensions 125352 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_cextensions 169566 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_nocextensions 171364 @@ -187,7 +187,7 @@ test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2. test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_postgresql_psycopg2_cextensions 18959 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_postgresql_psycopg2_nocextensions 19219 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_cextensions 22288 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 22530 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 21852 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_nocextensions 19492 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_cextensions 23067 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_nocextensions 23271 @@ -201,7 +201,7 @@ test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_mysql_mysqldb_nocexten test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_cextensions 1323 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_nocextensions 1348 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_cextensions 1601 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1626 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1603 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1355 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1656 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_nocextensions 1671 @@ -286,9 +286,9 @@ test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_mysql_mysqldb_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_mysql_mysqldb_nocextensions 80 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_cextensions 78 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_nocextensions 80 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_nocextensions 84 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_cextensions 78 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_nocextensions 80 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_nocextensions 84 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_cextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_nocextensions 78 test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_cextensions 78 @@ -320,9 +320,9 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_cextensions 514 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqldb_nocextensions 15534 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_cextensions 20501 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_nocextensions 35521 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_nocextensions 35528 test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 457 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_nocextensions 15477 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_nocextensions 15481 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 489 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_nocextensions 14489 test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 462 @@ -337,9 +337,9 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_ test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_cextensions 514 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqldb_nocextensions 45534 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_cextensions 20501 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_nocextensions 35521 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_nocextensions 35528 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 457 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_nocextensions 15477 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_nocextensions 15481 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 489 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_nocextensions 14489 test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 462 @@ -352,7 +352,7 @@ test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite # TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5892,292,3697,11893,1106,1968,2433 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5936,292,3929,13595,1223,2011,2692 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5936,295,3985,13782,1255,2064,2759 test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 5497,274,3609,11647,1097,1921,2486 test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 5519,274,3705,12819,1191,1928,2678 test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_cextensions 5497,273,3577,11529,1077,1883,2439 @@ -361,7 +361,7 @@ test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.4_postgresql_psyco # TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 6389,407,6826,18499,1134,2661 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 6480,412,7058,19930,1242,2726 +test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 6379,412,7054,19930,1258,2718 test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 6268,394,6860,18613,1107,2679 test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 6361,399,6964,19640,1193,2708 test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 3.4_postgresql_psycopg2_cextensions 6275,394,6860,18613,1107,2679 -- cgit v1.2.1 From a3241649332e988bd46c104e56f689dc03a2122d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Jan 2015 21:05:53 -0500 Subject: - this is passing, no idea why --- test/dialect/mysql/test_types.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/dialect/mysql/test_types.py b/test/dialect/mysql/test_types.py index 4e530e6b6..13425dc10 100644 --- a/test/dialect/mysql/test_types.py +++ b/test/dialect/mysql/test_types.py @@ -295,9 +295,6 @@ class TypesTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL): self.assert_compile(type_, expected) @testing.exclude('mysql', '<', (5, 0, 5), 'a 5.0+ feature') - @testing.fails_if( - lambda: testing.against("mysql+oursql") and util.py3k, - 'some round trips fail, oursql bug ?') @testing.provide_metadata def test_bit_50_roundtrip(self): bit_table = Table('mysql_bits', self.metadata, -- cgit v1.2.1 From 8257b3a0173b4ee2665f70c7e068497c462bd5df Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 5 Jan 2015 12:21:14 -0500 Subject: - more callcounts - add the platform key to the error output --- lib/sqlalchemy/testing/profiling.py | 6 ++++-- test/profiles.txt | 23 ++++++++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/sqlalchemy/testing/profiling.py b/lib/sqlalchemy/testing/profiling.py index 671bbe32d..57308925e 100644 --- a/lib/sqlalchemy/testing/profiling.py +++ b/lib/sqlalchemy/testing/profiling.py @@ -226,6 +226,7 @@ def count_functions(variance=0.05): callcount = stats.total_calls expected = _profile_stats.result(callcount) + if expected is None: expected_count = None else: @@ -249,10 +250,11 @@ def count_functions(variance=0.05): else: raise AssertionError( "Adjusted function call count %s not within %s%% " - "of expected %s. Rerun with --write-profiles to " + "of expected %s, platform %s. Rerun with " + "--write-profiles to " "regenerate this callcount." % ( callcount, (variance * 100), - expected_count)) + expected_count, _profile_stats.platform_key)) diff --git a/test/profiles.txt b/test/profiles.txt index b241653e6..0eb2add93 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -104,6 +104,7 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgre test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_postgresql_psycopg2_nocextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_cextensions 4265 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4262 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_cextensions 4263 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_nocextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_nocextensions 4266 @@ -118,6 +119,7 @@ test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_postgresql_psycopg2_nocextensions 6426 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_cextensions 6426 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_nocextensions 6426 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_cextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_nocextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_cextensions 6428 test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_nocextensions 6428 @@ -132,6 +134,7 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycop test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_nocextensions 40149 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 19280 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 28347 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_cextensions 20163 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_nocextensions 29138 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 20352 test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 29355 @@ -146,6 +149,7 @@ test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_nocextensions 30054 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 27144 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 28183 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_cextensions 26097 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_nocextensions 29068 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 26208 test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 31179 @@ -160,6 +164,7 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_postgresql_psycopg2_nocextensions 17988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_cextensions 17988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_nocextensions 17988 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_nocextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_cextensions 18988 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_nocextensions 18988 @@ -174,6 +179,7 @@ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_ test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_postgresql_psycopg2_nocextensions 122553 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_cextensions 162315 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 164551 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_cextensions 126351 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_nocextensions 125352 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_cextensions 169566 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_nocextensions 171364 @@ -188,6 +194,7 @@ test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2. test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_postgresql_psycopg2_nocextensions 19219 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_cextensions 22288 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 21852 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_cextensions 19423 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_nocextensions 19492 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_cextensions 23067 test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_nocextensions 23271 @@ -202,6 +209,7 @@ test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_ce test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_postgresql_psycopg2_nocextensions 1348 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_cextensions 1601 test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1603 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_cextensions 1354 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1355 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1656 test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_nocextensions 1671 @@ -210,17 +218,18 @@ test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_no # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_no_load -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_mysql_mysqldb_cextensions 117,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_mysql_mysqldb_nocextensions 117,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_cextensions 117,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_nocextensions 117,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_mysql_mysqldb_cextensions 91,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_mysql_mysqldb_nocextensions 91,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_cextensions 91,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_postgresql_psycopg2_nocextensions 91,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 91,18 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 91,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_nocextensions 122,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_cextensions 94,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_nocextensions 94,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 94,19 test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 94,19 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_cextensions 122,19 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_nocextensions 122,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_cextensions 94,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_nocextensions 94,19 # TEST: test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect -- cgit v1.2.1 From 41ae0270d99793608ce563b84e7befb3aa39252e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 5 Jan 2015 14:20:03 -0500 Subject: - make a new page that introduces mapping a little better for the "mapping" section, contrasts declarative and classical some more --- doc/build/orm/classical.rst | 67 +--------------------- doc/build/orm/mapper_config.rst | 2 +- doc/build/orm/mapping_styles.rst | 121 +++++++++++++++++++++++++++++++++++++++ doc/build/orm/relationships.rst | 5 +- 4 files changed, 126 insertions(+), 69 deletions(-) create mode 100644 doc/build/orm/mapping_styles.rst diff --git a/doc/build/orm/classical.rst b/doc/build/orm/classical.rst index 0f04586c7..3fd149f92 100644 --- a/doc/build/orm/classical.rst +++ b/doc/build/orm/classical.rst @@ -1,68 +1,5 @@ -.. _classical_mapping: +:orphan: -Classical Mappings -================== +Moved! :ref:`classical_mapping` -A *Classical Mapping* refers to the configuration of a mapped class using the -:func:`.mapper` function, without using the Declarative system. As an example, -start with the declarative mapping introduced in :ref:`ormtutorial_toplevel`:: - class User(Base): - __tablename__ = 'users' - - id = Column(Integer, primary_key=True) - name = Column(String) - fullname = Column(String) - password = Column(String) - -In "classical" form, the table metadata is created separately with the :class:`.Table` -construct, then associated with the ``User`` class via the :func:`.mapper` function:: - - from sqlalchemy import Table, MetaData, Column, ForeignKey, Integer, String - from sqlalchemy.orm import mapper - - metadata = MetaData() - - user = Table('user', metadata, - Column('id', Integer, primary_key=True), - Column('name', String(50)), - Column('fullname', String(50)), - Column('password', String(12)) - ) - - class User(object): - def __init__(self, name, fullname, password): - self.name = name - self.fullname = fullname - self.password = password - - mapper(User, user) - -Information about mapped attributes, such as relationships to other classes, are provided -via the ``properties`` dictionary. The example below illustrates a second :class:`.Table` -object, mapped to a class called ``Address``, then linked to ``User`` via :func:`.relationship`:: - - address = Table('address', metadata, - Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey('user.id')), - Column('email_address', String(50)) - ) - - mapper(User, user, properties={ - 'addresses' : relationship(Address, backref='user', order_by=address.c.id) - }) - - mapper(Address, address) - -When using classical mappings, classes must be provided directly without the benefit -of the "string lookup" system provided by Declarative. SQL expressions are typically -specified in terms of the :class:`.Table` objects, i.e. ``address.c.id`` above -for the ``Address`` relationship, and not ``Address.id``, as ``Address`` may not -yet be linked to table metadata, nor can we specify a string here. - -Some examples in the documentation still use the classical approach, but note that -the classical as well as Declarative approaches are **fully interchangeable**. Both -systems ultimately create the same configuration, consisting of a :class:`.Table`, -user-defined class, linked together with a :func:`.mapper`. When we talk about -"the behavior of :func:`.mapper`", this includes when using the Declarative system -as well - it's still used, just behind the scenes. diff --git a/doc/build/orm/mapper_config.rst b/doc/build/orm/mapper_config.rst index 9d584cbab..60ad7f5f9 100644 --- a/doc/build/orm/mapper_config.rst +++ b/doc/build/orm/mapper_config.rst @@ -13,9 +13,9 @@ know how to construct and use rudimentary mappers and relationships. .. toctree:: :maxdepth: 2 + mapping_styles scalar_mapping inheritance nonstandard_mappings - classical versioning mapping_api diff --git a/doc/build/orm/mapping_styles.rst b/doc/build/orm/mapping_styles.rst new file mode 100644 index 000000000..e6be00ef7 --- /dev/null +++ b/doc/build/orm/mapping_styles.rst @@ -0,0 +1,121 @@ +================= +Types of Mappings +================= + +Modern SQLAlchemy features two distinct styles of mapper configuration. +The "Classical" style is SQLAlchemy's original mapping API, whereas +"Declarative" is the richer and more succinct system that builds on top +of "Classical". Both styles may be used interchangeably, as the end +result of each is exactly the same - a user-defined class mapped by the +:func:`.mapper` function onto a selectable unit, typically a :class:`.Table`. + +Declarative Mapping +=================== + +The *Declarative Mapping* is the typical way that +mappings are constructed in modern SQLAlchemy. +Making use of the :ref:`declarative_toplevel` +system, the components of the user-defined class as well as the +:class:`.Table` metadata to which the class is mapped are defined +at once:: + + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy import Column, Integer, String, ForeignKey + + Base = declarative_base() + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + name = Column(String) + fullname = Column(String) + password = Column(String) + +Above, a basic single-table mapping with four columns. Additional +attributes, such as relationships to other mapped classes, are also +declared inline within the class definition:: + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + name = Column(String) + fullname = Column(String) + password = Column(String) + + addresses = relationship("Address", backref="user", order_by="Address.id") + + class Address(Base): + __tablename__ = 'address' + + id = Column(Integer, primary_key=True) + user_id = Column(ForeignKey('user.id')) + email_address = Column(String) + +The declarative mapping system is introduced in the +:ref:`ormtutorial_toplevel`. For additional details on how this system +works, see :ref:`declarative_toplevel`. + +.. _classical_mapping: + +Classical Mappings +================== + +A *Classical Mapping* refers to the configuration of a mapped class using the +:func:`.mapper` function, without using the Declarative system. This is +SQLAlchemy's original class mapping API, and is still the base mapping +system provided by the ORM. + +In "classical" form, the table metadata is created separately with the +:class:`.Table` construct, then associated with the ``User`` class via +the :func:`.mapper` function:: + + from sqlalchemy import Table, MetaData, Column, Integer, String, ForeignKey + from sqlalchemy.orm import mapper + + metadata = MetaData() + + user = Table('user', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)), + Column('fullname', String(50)), + Column('password', String(12)) + ) + + class User(object): + def __init__(self, name, fullname, password): + self.name = name + self.fullname = fullname + self.password = password + + mapper(User, user) + +Information about mapped attributes, such as relationships to other classes, are provided +via the ``properties`` dictionary. The example below illustrates a second :class:`.Table` +object, mapped to a class called ``Address``, then linked to ``User`` via :func:`.relationship`:: + + address = Table('address', metadata, + Column('id', Integer, primary_key=True), + Column('user_id', Integer, ForeignKey('user.id')), + Column('email_address', String(50)) + ) + + mapper(User, user, properties={ + 'addresses' : relationship(Address, backref='user', order_by=address.c.id) + }) + + mapper(Address, address) + +When using classical mappings, classes must be provided directly without the benefit +of the "string lookup" system provided by Declarative. SQL expressions are typically +specified in terms of the :class:`.Table` objects, i.e. ``address.c.id`` above +for the ``Address`` relationship, and not ``Address.id``, as ``Address`` may not +yet be linked to table metadata, nor can we specify a string here. + +Some examples in the documentation still use the classical approach, but note that +the classical as well as Declarative approaches are **fully interchangeable**. Both +systems ultimately create the same configuration, consisting of a :class:`.Table`, +user-defined class, linked together with a :func:`.mapper`. When we talk about +"the behavior of :func:`.mapper`", this includes when using the Declarative system +as well - it's still used, just behind the scenes. diff --git a/doc/build/orm/relationships.rst b/doc/build/orm/relationships.rst index 6fea107a7..f5cbac87e 100644 --- a/doc/build/orm/relationships.rst +++ b/doc/build/orm/relationships.rst @@ -6,9 +6,8 @@ Relationship Configuration ========================== This section describes the :func:`relationship` function and in depth discussion -of its usage. The reference material here continues into the next section, -:ref:`collections_toplevel`, which has additional detail on configuration -of collections via :func:`relationship`. +of its usage. For an introduction to relationships, start with the +:ref:`ormtutorial_toplevel` and head into :ref:`orm_tutorial_relationship`. .. toctree:: :maxdepth: 2 -- cgit v1.2.1 From 1104dcaa67062f27bf7519c8589f550bd5d5b4af Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 5 Jan 2015 19:02:08 -0500 Subject: - add MemoizedSlots, a generalized solution to using __getattr__ for memoization on a class that uses slots. - apply many more __slots__. mem use for nova now at 46% savings --- doc/build/changelog/migration_10.rst | 6 ++-- doc/build/core/metadata.rst | 1 + doc/build/orm/internals.rst | 25 +++++++++++++++++ lib/sqlalchemy/event/attr.py | 25 +++++++++-------- lib/sqlalchemy/ext/associationproxy.py | 2 +- lib/sqlalchemy/ext/hybrid.py | 4 +-- lib/sqlalchemy/orm/base.py | 7 ++++- lib/sqlalchemy/orm/descriptor_props.py | 4 +++ lib/sqlalchemy/orm/interfaces.py | 51 ++++++++++++++++++++++++++++------ lib/sqlalchemy/orm/mapper.py | 2 ++ lib/sqlalchemy/orm/properties.py | 20 +++++++++---- lib/sqlalchemy/orm/relationships.py | 1 + lib/sqlalchemy/orm/util.py | 10 ++++--- lib/sqlalchemy/sql/base.py | 21 ++++++++------ lib/sqlalchemy/sql/elements.py | 18 ++++++++---- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/_collections.py | 17 ++++++++++-- lib/sqlalchemy/util/langhelpers.py | 43 ++++++++++++++++++++++++++++ 18 files changed, 204 insertions(+), 55 deletions(-) diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index e5382be54..8870cfd7e 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -300,7 +300,7 @@ internals, comparator objects and parts of the ORM attribute and loader strategy system. A bench that makes use of heapy measure the startup size of Nova -illustrates a difference of about 2 megs of memory, a total of 27% +illustrates a difference of about 2 megs of memory, a total of 46% of memory taken up by SQLAlchemy's objects, associated dictionaries, as well as weakrefs, within a basic import of "nova.db.sqlalchemy.models":: @@ -308,13 +308,13 @@ well as weakrefs, within a basic import of "nova.db.sqlalchemy.models":: # associated dicts + weakref-related objects with core of Nova imported: Before: total count 26477 total bytes 7975712 - After: total count 21413 total bytes 5752976 + After: total count 18181 total bytes 4236456 # reported for the Python module space overall with the # core of Nova imported: Before: Partition of a set of 355558 objects. Total size = 61661760 bytes. - After: Partition of a set of 350281 objects. Total size = 59415104 bytes. + After: Partition of a set of 346034 objects. Total size = 57808016 bytes. .. _feature_updatemany: diff --git a/doc/build/core/metadata.rst b/doc/build/core/metadata.rst index d6fc8c6af..e46217c17 100644 --- a/doc/build/core/metadata.rst +++ b/doc/build/core/metadata.rst @@ -316,6 +316,7 @@ Column, Table, MetaData API .. autoclass:: SchemaItem :members: + :undoc-members: .. autoclass:: Table :members: diff --git a/doc/build/orm/internals.rst b/doc/build/orm/internals.rst index bead784a3..debb1ab7e 100644 --- a/doc/build/orm/internals.rst +++ b/doc/build/orm/internals.rst @@ -38,6 +38,8 @@ sections, are listed here. .. autoclass:: sqlalchemy.orm.base.InspectionAttr :members: +.. autoclass:: sqlalchemy.orm.base.InspectionAttrInfo + :members: .. autoclass:: sqlalchemy.orm.state.InstanceState :members: @@ -54,6 +56,29 @@ sections, are listed here. .. autoclass:: sqlalchemy.orm.interfaces.MapperProperty :members: + .. py:attribute:: info + + Info dictionary associated with the object, allowing user-defined + data to be associated with this :class:`.InspectionAttr`. + + The dictionary is generated when first accessed. Alternatively, + it can be specified as a constructor argument to the + :func:`.column_property`, :func:`.relationship`, or :func:`.composite` + functions. + + .. versionadded:: 0.8 Added support for .info to all + :class:`.MapperProperty` subclasses. + + .. versionchanged:: 1.0.0 :attr:`.InspectionAttr.info` moved + from :class:`.MapperProperty` so that it can apply to a wider + variety of ORM and extension constructs. + + .. seealso:: + + :attr:`.QueryableAttribute.info` + + :attr:`.SchemaItem.info` + .. autodata:: sqlalchemy.orm.interfaces.NOT_EXTENSION diff --git a/lib/sqlalchemy/event/attr.py b/lib/sqlalchemy/event/attr.py index de5d34950..ed1dca644 100644 --- a/lib/sqlalchemy/event/attr.py +++ b/lib/sqlalchemy/event/attr.py @@ -40,15 +40,19 @@ import weakref import collections -class RefCollection(object): - @util.memoized_property - def ref(self): +class RefCollection(util.MemoizedSlots): + __slots__ = 'ref', + + def _memoized_attr_ref(self): return weakref.ref(self, registry._collection_gced) class _ClsLevelDispatch(RefCollection): """Class-level events on :class:`._Dispatch` classes.""" + __slots__ = ('name', 'arg_names', 'has_kw', + 'legacy_signatures', '_clslevel') + def __init__(self, parent_dispatch_cls, fn): self.name = fn.__name__ argspec = util.inspect_getargspec(fn) @@ -60,8 +64,7 @@ class _ClsLevelDispatch(RefCollection): key=lambda s: s[0] ) )) - self.__doc__ = fn.__doc__ = legacy._augment_fn_docs( - self, parent_dispatch_cls, fn) + fn.__doc__ = legacy._augment_fn_docs(self, parent_dispatch_cls, fn) self._clslevel = weakref.WeakKeyDictionary() @@ -158,7 +161,7 @@ class _ClsLevelDispatch(RefCollection): return self -class _InstanceLevelDispatch(object): +class _InstanceLevelDispatch(RefCollection): __slots__ = () def _adjust_fn_spec(self, fn, named): @@ -229,10 +232,9 @@ class _EmptyListener(_InstanceLevelDispatch): class _CompoundListener(_InstanceLevelDispatch): _exec_once = False - __slots__ = () + __slots__ = '_exec_once_mutex', - @util.memoized_property - def _exec_once_mutex(self): + def _memoized_attr__exec_once_mutex(self): return threading.Lock() def exec_once(self, *args, **kw): @@ -267,7 +269,7 @@ class _CompoundListener(_InstanceLevelDispatch): __nonzero__ = __bool__ -class _ListenerCollection(RefCollection, _CompoundListener): +class _ListenerCollection(_CompoundListener): """Instance-level attributes on instances of :class:`._Dispatch`. Represents a collection of listeners. @@ -277,8 +279,7 @@ class _ListenerCollection(RefCollection, _CompoundListener): """ - # RefCollection has a @memoized_property, so can't do - # __slots__ here + __slots__ = 'parent_listeners', 'parent', 'name', 'listeners', 'propagate' def __init__(self, parent, target_cls): if target_cls not in parent._clslevel: diff --git a/lib/sqlalchemy/ext/associationproxy.py b/lib/sqlalchemy/ext/associationproxy.py index 1aa68ac32..bb08ce9ba 100644 --- a/lib/sqlalchemy/ext/associationproxy.py +++ b/lib/sqlalchemy/ext/associationproxy.py @@ -86,7 +86,7 @@ ASSOCIATION_PROXY = util.symbol('ASSOCIATION_PROXY') """ -class AssociationProxy(interfaces.InspectionAttr): +class AssociationProxy(interfaces.InspectionAttrInfo): """A descriptor that presents a read/write view of an object attribute.""" is_attribute = False diff --git a/lib/sqlalchemy/ext/hybrid.py b/lib/sqlalchemy/ext/hybrid.py index d89a13fc9..f72de6099 100644 --- a/lib/sqlalchemy/ext/hybrid.py +++ b/lib/sqlalchemy/ext/hybrid.py @@ -660,7 +660,7 @@ HYBRID_PROPERTY = util.symbol('HYBRID_PROPERTY') """ -class hybrid_method(interfaces.InspectionAttr): +class hybrid_method(interfaces.InspectionAttrInfo): """A decorator which allows definition of a Python object method with both instance-level and class-level behavior. @@ -703,7 +703,7 @@ class hybrid_method(interfaces.InspectionAttr): return self -class hybrid_property(interfaces.InspectionAttr): +class hybrid_property(interfaces.InspectionAttrInfo): """A decorator which allows definition of a Python descriptor with both instance-level and class-level behavior. diff --git a/lib/sqlalchemy/orm/base.py b/lib/sqlalchemy/orm/base.py index afeeba322..c5c8c5e2e 100644 --- a/lib/sqlalchemy/orm/base.py +++ b/lib/sqlalchemy/orm/base.py @@ -437,7 +437,6 @@ class InspectionAttr(object): here intact for forwards-compatibility. """ - __slots__ = () is_selectable = False @@ -490,6 +489,12 @@ class InspectionAttr(object): """ + +class InspectionAttrInfo(InspectionAttr): + """Adds the ``.info`` attribute to :class:`.Inspectionattr`. + + """ + @util.memoized_property def info(self): """Info dictionary associated with the object, allowing user-defined diff --git a/lib/sqlalchemy/orm/descriptor_props.py b/lib/sqlalchemy/orm/descriptor_props.py index 19ff71f73..e68ff1bea 100644 --- a/lib/sqlalchemy/orm/descriptor_props.py +++ b/lib/sqlalchemy/orm/descriptor_props.py @@ -143,6 +143,7 @@ class CompositeProperty(DescriptorProperty): class. **Deprecated.** Please see :class:`.AttributeEvents`. """ + super(CompositeProperty, self).__init__() self.attrs = attrs self.composite_class = class_ @@ -471,6 +472,7 @@ class ConcreteInheritedProperty(DescriptorProperty): return comparator_callable def __init__(self): + super(ConcreteInheritedProperty, self).__init__() def warn(): raise AttributeError("Concrete %s does not implement " "attribute %r at the instance level. Add " @@ -555,6 +557,7 @@ class SynonymProperty(DescriptorProperty): more complicated attribute-wrapping schemes than synonyms. """ + super(SynonymProperty, self).__init__() self.name = name self.map_column = map_column @@ -684,6 +687,7 @@ class ComparableProperty(DescriptorProperty): .. versionadded:: 1.0.0 """ + super(ComparableProperty, self).__init__() self.descriptor = descriptor self.comparator_factory = comparator_factory self.doc = doc or (descriptor and descriptor.__doc__) or None diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 68b86268c..346e2412e 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -24,7 +24,8 @@ from .. import util from ..sql import operators from .base import (ONETOMANY, MANYTOONE, MANYTOMANY, EXT_CONTINUE, EXT_STOP, NOT_EXTENSION) -from .base import InspectionAttr, _MappedAttribute +from .base import (InspectionAttr, InspectionAttr, + InspectionAttrInfo, _MappedAttribute) import collections # imported later @@ -48,7 +49,7 @@ __all__ = ( ) -class MapperProperty(_MappedAttribute, InspectionAttr): +class MapperProperty(_MappedAttribute, InspectionAttr, util.MemoizedSlots): """Manage the relationship of a ``Mapper`` to a single class attribute, as well as that attribute as it appears on individual instances of the class, including attribute instrumentation, @@ -63,6 +64,11 @@ class MapperProperty(_MappedAttribute, InspectionAttr): """ + __slots__ = ( + '_configure_started', '_configure_finished', 'parent', 'key', + 'info' + ) + cascade = frozenset() """The set of 'cascade' attribute names. @@ -78,6 +84,32 @@ class MapperProperty(_MappedAttribute, InspectionAttr): """ + def _memoized_attr_info(self): + """Info dictionary associated with the object, allowing user-defined + data to be associated with this :class:`.InspectionAttr`. + + The dictionary is generated when first accessed. Alternatively, + it can be specified as a constructor argument to the + :func:`.column_property`, :func:`.relationship`, or :func:`.composite` + functions. + + .. versionadded:: 0.8 Added support for .info to all + :class:`.MapperProperty` subclasses. + + .. versionchanged:: 1.0.0 :attr:`.InspectionAttr.info` moved + from :class:`.MapperProperty` so that it can apply to a wider + variety of ORM and extension constructs. + + .. seealso:: + + :attr:`.QueryableAttribute.info` + + :attr:`.SchemaItem.info` + + """ + return {} + + def setup(self, context, entity, path, adapter, **kwargs): """Called by Query for the purposes of constructing a SQL statement. @@ -139,8 +171,9 @@ class MapperProperty(_MappedAttribute, InspectionAttr): """ - _configure_started = False - _configure_finished = False + def __init__(self): + self._configure_started = False + self._configure_finished = False def init(self): """Called after all mappers are created to assemble @@ -422,6 +455,8 @@ class StrategizedProperty(MapperProperty): """ + __slots__ = '_strategies', 'strategy' + strategy_wildcard_key = None def _get_context_loader(self, context, path): @@ -485,14 +520,14 @@ class StrategizedProperty(MapperProperty): not mapper.class_manager._attr_has_impl(self.key): self.strategy.init_class_attribute(mapper) - _strategies = collections.defaultdict(dict) + _all_strategies = collections.defaultdict(dict) @classmethod def strategy_for(cls, **kw): def decorate(dec_cls): dec_cls._strategy_keys = [] key = tuple(sorted(kw.items())) - cls._strategies[cls][key] = dec_cls + cls._all_strategies[cls][key] = dec_cls dec_cls._strategy_keys.append(key) return dec_cls return decorate @@ -500,8 +535,8 @@ class StrategizedProperty(MapperProperty): @classmethod def _strategy_lookup(cls, *key): for prop_cls in cls.__mro__: - if prop_cls in cls._strategies: - strategies = cls._strategies[prop_cls] + if prop_cls in cls._all_strategies: + strategies = cls._all_strategies[prop_cls] try: return strategies[key] except KeyError: diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 9fe6b77f0..0469c2139 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -2787,6 +2787,8 @@ def _event_on_init(state, args, kwargs): class _ColumnMapping(dict): """Error reporting helper for mapper._columntoproperty.""" + __slots__ = 'mapper', + def __init__(self, mapper): self.mapper = mapper diff --git a/lib/sqlalchemy/orm/properties.py b/lib/sqlalchemy/orm/properties.py index 291fabdd0..d51b6920d 100644 --- a/lib/sqlalchemy/orm/properties.py +++ b/lib/sqlalchemy/orm/properties.py @@ -34,6 +34,13 @@ class ColumnProperty(StrategizedProperty): strategy_wildcard_key = 'column' + __slots__ = ( + '_orig_columns', 'columns', 'group', 'deferred', + 'instrument', 'comparator_factory', 'descriptor', 'extension', + 'active_history', 'expire_on_flush', 'info', 'doc', + 'strategy_class', '_creation_order', '_is_polymorphic_discriminator', + '_mapped_by_synonym') + def __init__(self, *columns, **kwargs): """Provide a column-level property for use with a Mapper. @@ -109,6 +116,7 @@ class ColumnProperty(StrategizedProperty): **Deprecated.** Please see :class:`.AttributeEvents`. """ + super(ColumnProperty, self).__init__() self._orig_columns = [expression._labeled(c) for c in columns] self.columns = [expression._labeled(_orm_full_deannotate(c)) for c in columns] @@ -206,7 +214,7 @@ class ColumnProperty(StrategizedProperty): elif dest_state.has_identity and self.key not in dest_dict: dest_state._expire_attributes(dest_dict, [self.key]) - class Comparator(PropComparator): + class Comparator(util.MemoizedSlots, PropComparator): """Produce boolean, comparison, and other operators for :class:`.ColumnProperty` attributes. @@ -225,8 +233,9 @@ class ColumnProperty(StrategizedProperty): """ - @util.memoized_instancemethod - def __clause_element__(self): + __slots__ = '__clause_element__', 'info' + + def _memoized_method___clause_element__(self): if self.adapter: return self.adapter(self.prop.columns[0]) else: @@ -234,15 +243,14 @@ class ColumnProperty(StrategizedProperty): "parententity": self._parentmapper, "parentmapper": self._parentmapper}) - @util.memoized_property - def info(self): + def _memoized_attr_info(self): ce = self.__clause_element__() try: return ce.info except AttributeError: return self.prop.info - def __getattr__(self, key): + def _fallback_getattr(self, key): """proxy attribute access down to the mapped column. this allows user-defined comparison methods to be accessed. diff --git a/lib/sqlalchemy/orm/relationships.py b/lib/sqlalchemy/orm/relationships.py index d3ae107b9..df2250a4c 100644 --- a/lib/sqlalchemy/orm/relationships.py +++ b/lib/sqlalchemy/orm/relationships.py @@ -775,6 +775,7 @@ class RelationshipProperty(StrategizedProperty): """ + super(RelationshipProperty, self).__init__() self.uselist = uselist self.argument = argument diff --git a/lib/sqlalchemy/orm/util.py b/lib/sqlalchemy/orm/util.py index 4be8d19ff..ee629b034 100644 --- a/lib/sqlalchemy/orm/util.py +++ b/lib/sqlalchemy/orm/util.py @@ -30,6 +30,10 @@ class CascadeOptions(frozenset): 'all', 'none', 'delete-orphan']) _allowed_cascades = all_cascades + __slots__ = ( + 'save_update', 'delete', 'refresh_expire', 'merge', + 'expunge', 'delete_orphan') + def __new__(cls, value_list): if isinstance(value_list, str) or value_list is None: return cls.from_string(value_list) @@ -38,10 +42,7 @@ class CascadeOptions(frozenset): raise sa_exc.ArgumentError( "Invalid cascade option(s): %s" % ", ".join([repr(x) for x in - sorted( - values.difference(cls._allowed_cascades) - )]) - ) + sorted(values.difference(cls._allowed_cascades))])) if "all" in values: values.update(cls._add_w_all_cascades) @@ -76,6 +77,7 @@ class CascadeOptions(frozenset): ] return cls(values) + def _validator_events( desc, key, validator, include_removes, include_backrefs): """Runs a validation method on an attribute value to be set or diff --git a/lib/sqlalchemy/sql/base.py b/lib/sqlalchemy/sql/base.py index 2d06109b9..0f6405309 100644 --- a/lib/sqlalchemy/sql/base.py +++ b/lib/sqlalchemy/sql/base.py @@ -449,10 +449,12 @@ class ColumnCollection(util.OrderedProperties): """ + __slots__ = '_all_col_set', '_all_columns' + def __init__(self, *columns): super(ColumnCollection, self).__init__() - self.__dict__['_all_col_set'] = util.column_set() - self.__dict__['_all_columns'] = [] + object.__setattr__(self, '_all_col_set', util.column_set()) + object.__setattr__(self, '_all_columns', []) for c in columns: self.add(c) @@ -576,13 +578,14 @@ class ColumnCollection(util.OrderedProperties): return util.OrderedProperties.__contains__(self, other) def __getstate__(self): - return {'_data': self.__dict__['_data'], - '_all_columns': self.__dict__['_all_columns']} + return {'_data': self._data, + '_all_columns': self._all_columns} def __setstate__(self, state): - self.__dict__['_data'] = state['_data'] - self.__dict__['_all_columns'] = state['_all_columns'] - self.__dict__['_all_col_set'] = util.column_set(state['_all_columns']) + object.__setattr__(self, '_data', state['_data']) + object.__setattr__(self, '_all_columns', state['_all_columns']) + object.__setattr__( + self, '_all_col_set', util.column_set(state['_all_columns'])) def contains_column(self, col): # this has to be done via set() membership @@ -596,8 +599,8 @@ class ColumnCollection(util.OrderedProperties): class ImmutableColumnCollection(util.ImmutableProperties, ColumnCollection): def __init__(self, data, colset, all_columns): util.ImmutableProperties.__init__(self, data) - self.__dict__['_all_col_set'] = colset - self.__dict__['_all_columns'] = all_columns + object.__setattr__(self, '_all_col_set', colset) + object.__setattr__(self, '_all_columns', all_columns) extend = remove = util.ImmutableProperties._immutable diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index 445857b82..8df22b7a6 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -3387,7 +3387,7 @@ class ReleaseSavepointClause(_IdentifiedClause): __visit_name__ = 'release_savepoint' -class quoted_name(util.text_type): +class quoted_name(util.MemoizedSlots, util.text_type): """Represent a SQL identifier combined with quoting preferences. :class:`.quoted_name` is a Python unicode/str subclass which @@ -3431,6 +3431,8 @@ class quoted_name(util.text_type): """ + __slots__ = 'quote', 'lower', 'upper' + def __new__(cls, value, quote): if value is None: return None @@ -3450,15 +3452,13 @@ class quoted_name(util.text_type): def __reduce__(self): return quoted_name, (util.text_type(self), self.quote) - @util.memoized_instancemethod - def lower(self): + def _memoized_method_lower(self): if self.quote: return self else: return util.text_type(self).lower() - @util.memoized_instancemethod - def upper(self): + def _memoized_method_upper(self): if self.quote: return self else: @@ -3475,6 +3475,8 @@ class _truncated_label(quoted_name): """A unicode subclass used to identify symbolic " "names that may require truncation.""" + __slots__ = () + def __new__(cls, value, quote=None): quote = getattr(value, "quote", quote) # return super(_truncated_label, cls).__new__(cls, value, quote, True) @@ -3531,6 +3533,7 @@ class conv(_truncated_label): :ref:`constraint_naming_conventions` """ + __slots__ = () class _defer_name(_truncated_label): @@ -3538,6 +3541,8 @@ class _defer_name(_truncated_label): generation. """ + __slots__ = () + def __new__(cls, value): if value is None: return _NONE_NAME @@ -3552,6 +3557,7 @@ class _defer_name(_truncated_label): class _defer_none_name(_defer_name): """indicate a 'deferred' name that was ultimately the value None.""" + __slots__ = () _NONE_NAME = _defer_none_name("_unnamed_") @@ -3566,6 +3572,8 @@ class _anonymous_label(_truncated_label): """A unicode subclass used to identify anonymously generated names.""" + __slots__ = () + def __add__(self, other): return _anonymous_label( quoted_name( diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 7c85ef94b..c23b0196f 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -36,7 +36,7 @@ from .langhelpers import iterate_attributes, class_hierarchy, \ generic_repr, counter, PluginLoader, hybridproperty, hybridmethod, \ safe_reraise,\ get_callable_argspec, only_once, attrsetter, ellipses_string, \ - warn_limited, map_bits + warn_limited, map_bits, MemoizedSlots from .deprecations import warn_deprecated, warn_pending_deprecation, \ deprecated, pending_deprecation, inject_docstring_text diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index d36852698..0f05e3427 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -179,8 +179,10 @@ class immutabledict(ImmutableContainer, dict): class Properties(object): """Provide a __getattr__/__setattr__ interface over a dict.""" + __slots__ = '_data', + def __init__(self, data): - self.__dict__['_data'] = data + object.__setattr__(self, '_data', data) def __len__(self): return len(self._data) @@ -200,8 +202,8 @@ class Properties(object): def __delitem__(self, key): del self._data[key] - def __setattr__(self, key, object): - self._data[key] = object + def __setattr__(self, key, obj): + self._data[key] = obj def __getstate__(self): return {'_data': self.__dict__['_data']} @@ -252,6 +254,8 @@ class OrderedProperties(Properties): """Provide a __getattr__/__setattr__ interface with an OrderedDict as backing store.""" + __slots__ = () + def __init__(self): Properties.__init__(self, OrderedDict()) @@ -259,10 +263,17 @@ class OrderedProperties(Properties): class ImmutableProperties(ImmutableContainer, Properties): """Provide immutable dict/object attribute to an underlying dictionary.""" + __slots__ = () + class OrderedDict(dict): """A dict that returns keys/values/items in the order they were added.""" + __slots__ = '_list', + + def __reduce__(self): + return OrderedDict, (self.items(),) + def __init__(self, ____sequence=None, **kwargs): self._list = [] if ____sequence is None: diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index b708665f9..22b6ad4ca 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -522,6 +522,15 @@ class portable_instancemethod(object): """ + __slots__ = 'target', 'name', '__weakref__' + + def __getstate__(self): + return {'target': self.target, 'name': self.name} + + def __setstate__(self, state): + self.target = state['target'] + self.name = state['name'] + def __init__(self, meth): self.target = meth.__self__ self.name = meth.__name__ @@ -800,6 +809,40 @@ class group_expirable_memoized_property(object): return memoized_instancemethod(fn) +class MemoizedSlots(object): + """Apply memoized items to an object using a __getattr__ scheme. + + This allows the functionality of memoized_property and + memoized_instancemethod to be available to a class using __slots__. + + """ + + def _fallback_getattr(self, key): + raise AttributeError(key) + + def __getattr__(self, key): + if key.startswith('_memoized'): + raise AttributeError(key) + elif hasattr(self, '_memoized_attr_%s' % key): + value = getattr(self, '_memoized_attr_%s' % key)() + setattr(self, key, value) + return value + elif hasattr(self, '_memoized_method_%s' % key): + fn = getattr(self, '_memoized_method_%s' % key) + + def oneshot(*args, **kw): + result = fn(*args, **kw) + memo = lambda *a, **kw: result + memo.__name__ = fn.__name__ + memo.__doc__ = fn.__doc__ + setattr(self, key, memo) + return result + oneshot.__doc__ = fn.__doc__ + return oneshot + else: + return self._fallback_getattr(key) + + def dependency_for(modulename): def decorate(obj): # TODO: would be nice to improve on this import silliness, -- cgit v1.2.1 From 145d0151515d8119931eb2c79425a4e38eb6cae4 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 5 Jan 2015 19:07:33 -0500 Subject: fix verbiage --- doc/build/changelog/migration_10.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 8870cfd7e..52179a323 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -300,8 +300,8 @@ internals, comparator objects and parts of the ORM attribute and loader strategy system. A bench that makes use of heapy measure the startup size of Nova -illustrates a difference of about 2 megs of memory, a total of 46% -of memory taken up by SQLAlchemy's objects, associated dictionaries, as +illustrates a difference of about 3.7 fewer megs, or 46%, +taken up by SQLAlchemy's objects, associated dictionaries, as well as weakrefs, within a basic import of "nova.db.sqlalchemy.models":: # reported by heapy, summation of SQLAlchemy objects + -- cgit v1.2.1 From 57f684b4b4a661c78de0b5953603984714f01e0b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 5 Jan 2015 21:38:19 -0500 Subject: - Fixed bug where if an exception were thrown at the start of a :class:`.Query` before it fetched results, particularly when row processors can't be formed, the cursor would stay open with results pending and not actually be closed. This is typically only an issue on an interpreter like Pypy where the cursor isn't immediately GC'ed, and can in some circumstances lead to transactions/ locks being open longer than is desirable. fixes #3285 --- doc/build/changelog/changelog_09.rst | 13 +++++++ lib/sqlalchemy/orm/loading.py | 66 +++++++++++++++++++----------------- test/orm/test_loading.py | 33 ++++++++++++++++-- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index e0f46eb66..81a26d187 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -14,6 +14,19 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, orm, pypy + :versions: 1.0.0 + :tickets: 3285 + + Fixed bug where if an exception were thrown at the start of a + :class:`.Query` before it fetched results, particularly when + row processors can't be formed, the cursor would stay open with + results pending and not actually be closed. This is typically only + an issue on an interpreter like Pypy where the cursor isn't + immediately GC'ed, and can in some circumstances lead to transactions/ + locks being open longer than is desirable. + .. change:: :tags: change, mysql :versions: 1.0.0 diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py index 380afcdc7..fdc787545 100644 --- a/lib/sqlalchemy/orm/loading.py +++ b/lib/sqlalchemy/orm/loading.py @@ -42,41 +42,45 @@ def instances(query, cursor, context): def filter_fn(row): return tuple(fn(x) for x, fn in zip(row, filter_fns)) - (process, labels) = \ - list(zip(*[ - query_entity.row_processor(query, - context, cursor) - for query_entity in query._entities - ])) - - if not single_entity: - keyed_tuple = util.lightweight_named_tuple('result', labels) - - while True: - context.partials = {} - - if query._yield_per: - fetch = cursor.fetchmany(query._yield_per) - if not fetch: - break - else: - fetch = cursor.fetchall() + try: + (process, labels) = \ + list(zip(*[ + query_entity.row_processor(query, + context, cursor) + for query_entity in query._entities + ])) + + if not single_entity: + keyed_tuple = util.lightweight_named_tuple('result', labels) + + while True: + context.partials = {} + + if query._yield_per: + fetch = cursor.fetchmany(query._yield_per) + if not fetch: + break + else: + fetch = cursor.fetchall() - if single_entity: - proc = process[0] - rows = [proc(row) for row in fetch] - else: - rows = [keyed_tuple([proc(row) for proc in process]) - for row in fetch] + if single_entity: + proc = process[0] + rows = [proc(row) for row in fetch] + else: + rows = [keyed_tuple([proc(row) for proc in process]) + for row in fetch] - if filtered: - rows = util.unique_list(rows, filter_fn) + if filtered: + rows = util.unique_list(rows, filter_fn) - for row in rows: - yield row + for row in rows: + yield row - if not query._yield_per: - break + if not query._yield_per: + break + except Exception as err: + cursor.close() + util.raise_from_cause(err) @util.dependencies("sqlalchemy.orm.query") diff --git a/test/orm/test_loading.py b/test/orm/test_loading.py index 97c08ea29..f86477ec2 100644 --- a/test/orm/test_loading.py +++ b/test/orm/test_loading.py @@ -1,13 +1,40 @@ from . import _fixtures from sqlalchemy.orm import loading, Session, aliased -from sqlalchemy.testing.assertions import eq_ +from sqlalchemy.testing.assertions import eq_, assert_raises from sqlalchemy.util import KeyedTuple - -# class InstancesTest(_fixtures.FixtureTest): +from sqlalchemy.testing import mock # class GetFromIdentityTest(_fixtures.FixtureTest): # class LoadOnIdentTest(_fixtures.FixtureTest): # class InstanceProcessorTest(_fixture.FixtureTest): + +class InstancesTest(_fixtures.FixtureTest): + run_setup_mappers = 'once' + run_inserts = 'once' + run_deletes = None + + @classmethod + def setup_mappers(cls): + cls._setup_stock_mapping() + + def test_cursor_close_w_failed_rowproc(self): + User = self.classes.User + s = Session() + + q = s.query(User) + + ctx = q._compile_context() + cursor = mock.Mock() + q._entities = [ + mock.Mock(row_processor=mock.Mock(side_effect=Exception("boom"))) + ] + assert_raises( + Exception, + list, loading.instances(q, cursor, ctx) + ) + assert cursor.close.called, "Cursor wasn't closed" + + class MergeResultTest(_fixtures.FixtureTest): run_setup_mappers = 'once' run_inserts = 'once' -- cgit v1.2.1 From 67778e20624bfb63990ff67598f911768867e439 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 6 Jan 2015 11:34:52 -0500 Subject: - add a close here --- test/sql/test_join_rewriting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/sql/test_join_rewriting.py b/test/sql/test_join_rewriting.py index ced65d7f1..f99dfda4e 100644 --- a/test/sql/test_join_rewriting.py +++ b/test/sql/test_join_rewriting.py @@ -650,6 +650,7 @@ class JoinExecTest(_JoinRewriteTestBase, fixtures.TestBase): def _test(self, selectable, assert_): result = testing.db.execute(selectable) + result.close() for col in selectable.inner_columns: assert col in result._metadata._keymap -- cgit v1.2.1 From b8a8cdd1ff47b5774662f4c61fe49382b967de02 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 6 Jan 2015 11:45:17 -0500 Subject: - doc fixes --- lib/sqlalchemy/orm/base.py | 13 +++++++++---- lib/sqlalchemy/orm/interfaces.py | 13 +++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/sqlalchemy/orm/base.py b/lib/sqlalchemy/orm/base.py index c5c8c5e2e..7bfafdc2b 100644 --- a/lib/sqlalchemy/orm/base.py +++ b/lib/sqlalchemy/orm/base.py @@ -491,7 +491,11 @@ class InspectionAttr(object): class InspectionAttrInfo(InspectionAttr): - """Adds the ``.info`` attribute to :class:`.Inspectionattr`. + """Adds the ``.info`` attribute to :class:`.InspectionAttr`. + + The rationale for :class:`.InspectionAttr` vs. :class:`.InspectionAttrInfo` + is that the former is compatible as a mixin for classes that specify + ``__slots__``; this is essentially an implementation artifact. """ @@ -508,9 +512,10 @@ class InspectionAttrInfo(InspectionAttr): .. versionadded:: 0.8 Added support for .info to all :class:`.MapperProperty` subclasses. - .. versionchanged:: 1.0.0 :attr:`.InspectionAttr.info` moved - from :class:`.MapperProperty` so that it can apply to a wider - variety of ORM and extension constructs. + .. versionchanged:: 1.0.0 :attr:`.MapperProperty.info` is also + available on extension types via the + :attr:`.InspectionAttrInfo.info` attribute, so that it can apply + to a wider variety of ORM and extension constructs. .. seealso:: diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 346e2412e..299ccaaaf 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -50,10 +50,7 @@ __all__ = ( class MapperProperty(_MappedAttribute, InspectionAttr, util.MemoizedSlots): - """Manage the relationship of a ``Mapper`` to a single class - attribute, as well as that attribute as it appears on individual - instances of the class, including attribute instrumentation, - attribute access, loading behavior, and dependency calculations. + """Represent a particular class attribute mapped by :class:`.Mapper`. The most common occurrences of :class:`.MapperProperty` are the mapped :class:`.Column`, which is represented in a mapping as @@ -96,9 +93,10 @@ class MapperProperty(_MappedAttribute, InspectionAttr, util.MemoizedSlots): .. versionadded:: 0.8 Added support for .info to all :class:`.MapperProperty` subclasses. - .. versionchanged:: 1.0.0 :attr:`.InspectionAttr.info` moved - from :class:`.MapperProperty` so that it can apply to a wider - variety of ORM and extension constructs. + .. versionchanged:: 1.0.0 :attr:`.MapperProperty.info` is also + available on extension types via the + :attr:`.InspectionAttrInfo.info` attribute, so that it can apply + to a wider variety of ORM and extension constructs. .. seealso:: @@ -109,7 +107,6 @@ class MapperProperty(_MappedAttribute, InspectionAttr, util.MemoizedSlots): """ return {} - def setup(self, context, entity, path, adapter, **kwargs): """Called by Query for the purposes of constructing a SQL statement. -- cgit v1.2.1 From f4b7b02e31e6b49195c21da7221bcbda0bad02b9 Mon Sep 17 00:00:00 2001 From: Dimitris Theodorou Date: Mon, 12 Jan 2015 02:40:50 +0100 Subject: Add native_enum flag to Enum's repr() result Needed for alembic autogenerate rendering. --- lib/sqlalchemy/sql/sqltypes.py | 1 + test/sql/test_types.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 9b0d26601..bd1914da3 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -1146,6 +1146,7 @@ class Enum(String, SchemaType): def __repr__(self): return util.generic_repr(self, + additional_kw=[('native_enum', True)], to_inspect=[Enum, SchemaType], ) diff --git a/test/sql/test_types.py b/test/sql/test_types.py index 26dc6c842..0212499c4 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -1157,8 +1157,8 @@ class EnumTest(AssertsCompiledSQL, fixtures.TestBase): def test_repr(self): e = Enum( "x", "y", name="somename", convert_unicode=True, quote=True, - inherit_schema=True) - eq_(repr(e), "Enum('x', 'y', name='somename', inherit_schema=True)") + inherit_schema=True, native_enum=False) + eq_(repr(e), "Enum('x', 'y', name='somename', inherit_schema=True, native_enum=False)") binary_table = MyPickleType = metadata = None -- cgit v1.2.1 From 5f1d34c4c86263684d5a79c8d8f9db8d1e3afccb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 12 Jan 2015 13:24:11 -0500 Subject: - changelog for pr 41 --- doc/build/changelog/changelog_09.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 81a26d187..b675be3b0 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -14,6 +14,15 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, sql + :versions: 1.0.0 + :pullreq: bitbucket:41 + + Added the ``native_enum`` flag to the ``__repr__()`` output + of :class:`.Enum`, which is mostly important when using it with + Alembic autogenerate. Pull request courtesy Dimitris Theodorou. + .. change:: :tags: bug, orm, pypy :versions: 1.0.0 -- cgit v1.2.1 From dc55ff6f99098450f20aa702a55ece30b7e5fc7c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 12 Jan 2015 13:27:34 -0500 Subject: repair formatting --- test/sql/test_types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/sql/test_types.py b/test/sql/test_types.py index 0212499c4..6ffd88d78 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -1158,7 +1158,10 @@ class EnumTest(AssertsCompiledSQL, fixtures.TestBase): e = Enum( "x", "y", name="somename", convert_unicode=True, quote=True, inherit_schema=True, native_enum=False) - eq_(repr(e), "Enum('x', 'y', name='somename', inherit_schema=True, native_enum=False)") + eq_( + repr(e), + "Enum('x', 'y', name='somename', " + "inherit_schema=True, native_enum=False)") binary_table = MyPickleType = metadata = None -- cgit v1.2.1 From 92cc232726a01dd3beff762ebccd326a9659e8b9 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 13 Jan 2015 14:33:33 -0500 Subject: - The multi-values version of :meth:`.Insert.values` has been repaired to work more usefully with tables that have Python- side default values and/or functions, as well as server-side defaults. The feature will now work with a dialect that uses "positional" parameters; a Python callable will also be invoked individually for each row just as is the case with an "executemany" style invocation; a server- side default column will no longer implicitly receive the value explicitly specified for the first row, instead refusing to invoke without an explicit value. fixes #3288 --- doc/build/changelog/changelog_10.rst | 19 +++++ doc/build/changelog/migration_10.rst | 83 +++++++++++++++++++++ lib/sqlalchemy/engine/default.py | 9 +-- lib/sqlalchemy/sql/crud.py | 83 ++++++++++++--------- lib/sqlalchemy/sql/dml.py | 6 ++ lib/sqlalchemy/testing/assertions.py | 3 + test/sql/test_defaults.py | 22 ++++-- test/sql/test_insert.py | 136 +++++++++++++++++++++++++++++------ 8 files changed, 296 insertions(+), 65 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 7f9fbff91..5d8bb7b68 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,25 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, sql + :tickets: 3288 + + The multi-values version of :meth:`.Insert.values` has been + repaired to work more usefully with tables that have Python- + side default values and/or functions, as well as server-side + defaults. The feature will now work with a dialect that uses + "positional" parameters; a Python callable will also be + invoked individually for each row just as is the case with an + "executemany" style invocation; a server- side default column + will no longer implicitly receive the value explicitly + specified for the first row, instead refusing to invoke + without an explicit value. + + .. seealso:: + + :ref:`bug_3288` + .. change:: :tags: feature, general diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 52179a323..bd878f4cb 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1362,6 +1362,89 @@ be qualified with :func:`.text` or similar. :ticket:`2992` +.. _bug_3288: + +Python-side defaults invoked for each row invidually when using a multivalued insert +------------------------------------------------------------------------------------ + +Support for Python-side column defaults when using the multi-valued +version of :meth:`.Insert.values` were essentially not implemented, and +would only work "by accident" in specific situations, when the dialect in +use was using a non-positional (e.g. named) style of bound parameter, and +when it was not necessary that a Python-side callable be invoked for each +row. + +The feature has been overhauled so that it works more similarly to +that of an "executemany" style of invocation:: + + import itertools + + counter = itertools.count(1) + t = Table( + 'my_table', metadata, + Column('id', Integer, default=lambda: next(counter)), + Column('data', String) + ) + + conn.execute(t.insert().values([ + {"data": "d1"}, + {"data": "d2"}, + {"data": "d3"}, + ])) + +The above example will invoke ``next(counter)`` for each row individually +as would be expected:: + + INSERT INTO my_table (id, data) VALUES (?, ?), (?, ?), (?, ?) + (1, 'd1', 2, 'd2', 3, 'd3') + +Previously, a positional dialect would fail as a bind would not be generated +for additional positions:: + + Incorrect number of bindings supplied. The current statement uses 6, + and there are 4 supplied. + [SQL: u'INSERT INTO my_table (id, data) VALUES (?, ?), (?, ?), (?, ?)'] + [parameters: (1, 'd1', 'd2', 'd3')] + +And with a "named" dialect, the same value for "id" would be re-used in +each row (hence this change is backwards-incompatible with a system that +relied on this):: + + INSERT INTO my_table (id, data) VALUES (:id, :data_0), (:id, :data_1), (:id, :data_2) + {u'data_2': 'd3', u'data_1': 'd2', u'data_0': 'd1', 'id': 1} + +The system will also refuse to invoke a "server side" default as inline-rendered +SQL, since it cannot be guaranteed that a server side default is compatible +with this. If the VALUES clause renders for a specific column, then a Python-side +value is required; if an omitted value only refers to a server-side default, +an exception is raised:: + + t = Table( + 'my_table', metadata, + Column('id', Integer, primary_key=True), + Column('data', String, server_default='some default') + ) + + conn.execute(t.insert().values([ + {"data": "d1"}, + {"data": "d2"}, + {}, + ])) + +will raise:: + + sqlalchemy.exc.CompileError: INSERT value for column my_table.data is + explicitly rendered as a boundparameter in the VALUES clause; a + Python-side value or SQL expression is required + +Previously, the value "d1" would be copied into that of the third +row (but again, only with named format!):: + + INSERT INTO my_table (data) VALUES (:data_0), (:data_1), (:data_0) + {u'data_1': 'd2', u'data_0': 'd1'} + +:ticket:`3288` + .. _change_3163: Event listeners can not be added or removed from within that event's runner diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index a5af6ff19..c5b5deece 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -956,14 +956,17 @@ class DefaultExecutionContext(interfaces.ExecutionContext): def _process_executesingle_defaults(self): key_getter = self.compiled._key_getters_for_crud_column[2] - prefetch = self.compiled.prefetch self.current_parameters = compiled_parameters = \ self.compiled_parameters[0] for c in prefetch: if self.isinsert: - val = self.get_insert_default(c) + if c.default and \ + not c.default.is_sequence and c.default.is_scalar: + val = c.default.arg + else: + val = self.get_insert_default(c) else: val = self.get_update_default(c) @@ -972,6 +975,4 @@ class DefaultExecutionContext(interfaces.ExecutionContext): del self.current_parameters - - DefaultDialect.execution_ctx_cls = DefaultExecutionContext diff --git a/lib/sqlalchemy/sql/crud.py b/lib/sqlalchemy/sql/crud.py index 831d05be1..4bab69df0 100644 --- a/lib/sqlalchemy/sql/crud.py +++ b/lib/sqlalchemy/sql/crud.py @@ -116,11 +116,13 @@ def _get_crud_params(compiler, stmt, **kw): def _create_bind_param( - compiler, col, value, process=True, required=False, name=None): + compiler, col, value, process=True, + required=False, name=None, unique=False): if name is None: name = col.key - bindparam = elements.BindParameter(name, value, - type_=col.type, required=required) + bindparam = elements.BindParameter( + name, value, + type_=col.type, required=required, unique=unique) bindparam._is_crud = True if process: bindparam = bindparam._compiler_dispatch(compiler) @@ -299,14 +301,49 @@ def _append_param_insert_pk_returning(compiler, stmt, c, values, kw): ) compiler.returning.append(c) else: - values.append( - (c, _create_bind_param(compiler, c, None)) - ) - compiler.prefetch.append(c) + _create_prefetch_bind_param(compiler, c, values) else: compiler.returning.append(c) +def _create_prefetch_bind_param(compiler, c, values, process=True, name=None): + values.append( + (c, _create_bind_param(compiler, c, None, process=process, name=name)) + ) + compiler.prefetch.append(c) + + +class _multiparam_column(elements.ColumnElement): + def __init__(self, original, index): + self.key = "%s_%d" % (original.key, index + 1) + self.original = original + self.default = original.default + + def __eq__(self, other): + return isinstance(other, _multiparam_column) and \ + other.key == self.key and \ + other.original == self.original + + +def _process_multiparam_default_bind( + compiler, c, index, kw): + + if not c.default: + raise exc.CompileError( + "INSERT value for column %s is explicitly rendered as a bound" + "parameter in the VALUES clause; " + "a Python-side value or SQL expression is required" % c) + elif c.default.is_clause_element: + return compiler.process(c.default.arg.self_group(), **kw) + else: + col = _multiparam_column(c, index) + bind = _create_bind_param( + compiler, col, None + ) + compiler.prefetch.append(col) + return bind + + def _append_param_insert_pk(compiler, stmt, c, values, kw): if ( (c.default is not None and @@ -317,11 +354,7 @@ def _append_param_insert_pk(compiler, stmt, c, values, kw): compiler.dialect. preexecute_autoincrement_sequences) ): - values.append( - (c, _create_bind_param(compiler, c, None)) - ) - - compiler.prefetch.append(c) + _create_prefetch_bind_param(compiler, c, values) def _append_param_insert_hasdefault( @@ -349,10 +382,7 @@ def _append_param_insert_hasdefault( # don't add primary key column to postfetch compiler.postfetch.append(c) else: - values.append( - (c, _create_bind_param(compiler, c, None)) - ) - compiler.prefetch.append(c) + _create_prefetch_bind_param(compiler, c, values) def _append_param_insert_select_hasdefault( @@ -368,10 +398,7 @@ def _append_param_insert_select_hasdefault( proc = c.default.arg.self_group() values.append((c, proc)) else: - values.append( - (c, _create_bind_param(compiler, c, None, process=False)) - ) - compiler.prefetch.append(c) + _create_prefetch_bind_param(compiler, c, values, process=False) def _append_param_update( @@ -389,10 +416,7 @@ def _append_param_update( else: compiler.postfetch.append(c) else: - values.append( - (c, _create_bind_param(compiler, c, None)) - ) - compiler.prefetch.append(c) + _create_prefetch_bind_param(compiler, c, values) elif c.server_onupdate is not None: if implicit_return_defaults and \ c in implicit_return_defaults: @@ -444,13 +468,7 @@ def _get_multitable_params( ) compiler.postfetch.append(c) else: - values.append( - (c, _create_bind_param( - compiler, c, None, name=_col_bind_name(c) - ) - ) - ) - compiler.prefetch.append(c) + _create_prefetch_bind_param(compiler, c, values, name=_col_bind_name(c)) elif c.server_onupdate is not None: compiler.postfetch.append(c) @@ -469,7 +487,8 @@ def _extend_values_for_multiparams(compiler, stmt, values, kw): ) if elements._is_literal(row[c.key]) else compiler.process( row[c.key].self_group(), **kw)) - if c.key in row else param + if c.key in row else + _process_multiparam_default_bind(compiler, c, i, kw) ) for (c, param) in values_0 ] diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py index 62169319b..38b3b8c44 100644 --- a/lib/sqlalchemy/sql/dml.py +++ b/lib/sqlalchemy/sql/dml.py @@ -277,6 +277,12 @@ class ValuesBase(UpdateBase): deals with an arbitrary number of rows, so the :attr:`.ResultProxy.inserted_primary_key` accessor does not apply. + .. versionchanged:: 1.0.0 A multiple-VALUES INSERT now supports + columns with Python side default values and callables in the + same way as that of an "executemany" style of invocation; the + callable is invoked for each row. See :ref:`bug_3288` + for other details. + .. seealso:: :ref:`inserts_and_updates` - SQL Expression diff --git a/lib/sqlalchemy/testing/assertions.py b/lib/sqlalchemy/testing/assertions.py index 66d1f3cb0..46fcd64b1 100644 --- a/lib/sqlalchemy/testing/assertions.py +++ b/lib/sqlalchemy/testing/assertions.py @@ -229,6 +229,7 @@ class AssertsCompiledSQL(object): def assert_compile(self, clause, result, params=None, checkparams=None, dialect=None, checkpositional=None, + check_prefetch=None, use_default_dialect=False, allow_dialect_select=False, literal_binds=False): @@ -289,6 +290,8 @@ class AssertsCompiledSQL(object): if checkpositional is not None: p = c.construct_params(params) eq_(tuple([p[x] for x in c.positiontup]), checkpositional) + if check_prefetch is not None: + eq_(c.prefetch, check_prefetch) class ComparesTables(object): diff --git a/test/sql/test_defaults.py b/test/sql/test_defaults.py index 10e557b76..b7893d5f1 100644 --- a/test/sql/test_defaults.py +++ b/test/sql/test_defaults.py @@ -336,13 +336,7 @@ class DefaultTest(fixtures.TestBase): [(54, 'imthedefault', f, ts, ts, ctexec, True, False, 12, today, None, 'hi')]) - @testing.fails_on('firebird', 'Data type unknown') def test_insertmany(self): - # MySQL-Python 1.2.2 breaks functions in execute_many :( - if (testing.against('mysql+mysqldb') and - testing.db.dialect.dbapi.version_info[:3] == (1, 2, 2)): - return - t.insert().execute({}, {}, {}) ctexec = currenttime.scalar() @@ -356,6 +350,22 @@ class DefaultTest(fixtures.TestBase): (53, 'imthedefault', f, ts, ts, ctexec, True, False, 12, today, 'py', 'hi')]) + @testing.requires.multivalues_inserts + def test_insert_multivalues(self): + + t.insert().values([{}, {}, {}]).execute() + + ctexec = currenttime.scalar() + l = t.select().execute() + today = datetime.date.today() + eq_(l.fetchall(), + [(51, 'imthedefault', f, ts, ts, ctexec, True, False, + 12, today, 'py', 'hi'), + (52, 'imthedefault', f, ts, ts, ctexec, True, False, + 12, today, 'py', 'hi'), + (53, 'imthedefault', f, ts, ts, ctexec, True, False, + 12, today, 'py', 'hi')]) + def test_no_embed_in_sql(self): """Using a DefaultGenerator, Sequence, DefaultClause in the columns, where clause of a select, or in the values diff --git a/test/sql/test_insert.py b/test/sql/test_insert.py index bd4eaa3e2..8a41d4be7 100644 --- a/test/sql/test_insert.py +++ b/test/sql/test_insert.py @@ -1,12 +1,12 @@ #! coding:utf-8 from sqlalchemy import Column, Integer, MetaData, String, Table,\ - bindparam, exc, func, insert, select, column + bindparam, exc, func, insert, select, column, text from sqlalchemy.dialects import mysql, postgresql from sqlalchemy.engine import default from sqlalchemy.testing import AssertsCompiledSQL,\ assert_raises_message, fixtures - +from sqlalchemy.sql import crud class _InsertTestBase(object): @@ -19,6 +19,12 @@ class _InsertTestBase(object): Table('myothertable', metadata, Column('otherid', Integer, primary_key=True), Column('othername', String(30))) + Table('table_w_defaults', metadata, + Column('id', Integer, primary_key=True), + Column('x', Integer, default=10), + Column('y', Integer, server_default=text('5')), + Column('z', Integer, default=lambda: 10) + ) class InsertTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): @@ -565,6 +571,36 @@ class MultirowTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): checkpositional=checkpositional, dialect=dialect) + def test_positional_w_defaults(self): + table1 = self.tables.table_w_defaults + + values = [ + {'id': 1}, + {'id': 2}, + {'id': 3} + ] + + checkpositional = (1, None, None, 2, None, None, 3, None, None) + + dialect = default.DefaultDialect() + dialect.supports_multivalues_insert = True + dialect.paramstyle = 'format' + dialect.positional = True + + self.assert_compile( + table1.insert().values(values), + "INSERT INTO table_w_defaults (id, x, z) VALUES " + "(%s, %s, %s), (%s, %s, %s), (%s, %s, %s)", + checkpositional=checkpositional, + check_prefetch=[ + table1.c.x, table1.c.z, + crud._multiparam_column(table1.c.x, 0), + crud._multiparam_column(table1.c.z, 0), + crud._multiparam_column(table1.c.x, 1), + crud._multiparam_column(table1.c.z, 1) + ], + dialect=dialect) + def test_inline_default(self): metadata = MetaData() table = Table('sometable', metadata, @@ -597,6 +633,74 @@ class MultirowTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): checkparams=checkparams, dialect=postgresql.dialect()) + def test_python_scalar_default(self): + metadata = MetaData() + table = Table('sometable', metadata, + Column('id', Integer, primary_key=True), + Column('data', String), + Column('foo', Integer, default=10)) + + values = [ + {'id': 1, 'data': 'data1'}, + {'id': 2, 'data': 'data2', 'foo': 15}, + {'id': 3, 'data': 'data3'}, + ] + + checkparams = { + 'id_0': 1, + 'id_1': 2, + 'id_2': 3, + 'data_0': 'data1', + 'data_1': 'data2', + 'data_2': 'data3', + 'foo': None, # evaluated later + 'foo_1': 15, + 'foo_2': None # evaluated later + } + + self.assert_compile( + table.insert().values(values), + 'INSERT INTO sometable (id, data, foo) VALUES ' + '(%(id_0)s, %(data_0)s, %(foo)s), ' + '(%(id_1)s, %(data_1)s, %(foo_1)s), ' + '(%(id_2)s, %(data_2)s, %(foo_2)s)', + checkparams=checkparams, + dialect=postgresql.dialect()) + + def test_python_fn_default(self): + metadata = MetaData() + table = Table('sometable', metadata, + Column('id', Integer, primary_key=True), + Column('data', String), + Column('foo', Integer, default=lambda: 10)) + + values = [ + {'id': 1, 'data': 'data1'}, + {'id': 2, 'data': 'data2', 'foo': 15}, + {'id': 3, 'data': 'data3'}, + ] + + checkparams = { + 'id_0': 1, + 'id_1': 2, + 'id_2': 3, + 'data_0': 'data1', + 'data_1': 'data2', + 'data_2': 'data3', + 'foo': None, # evaluated later + 'foo_1': 15, + 'foo_2': None, # evaluated later + } + + self.assert_compile( + table.insert().values(values), + "INSERT INTO sometable (id, data, foo) VALUES " + "(%(id_0)s, %(data_0)s, %(foo)s), " + "(%(id_1)s, %(data_1)s, %(foo_1)s), " + "(%(id_2)s, %(data_2)s, %(foo_2)s)", + checkparams=checkparams, + dialect=postgresql.dialect()) + def test_sql_functions(self): metadata = MetaData() table = Table('sometable', metadata, @@ -684,24 +788,10 @@ class MultirowTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): {'id': 3, 'data': 'data3', 'foo': 'otherfoo'}, ] - checkparams = { - 'id_0': 1, - 'id_1': 2, - 'id_2': 3, - 'data_0': 'data1', - 'data_1': 'data2', - 'data_2': 'data3', - 'foo_0': 'plainfoo', - 'foo_2': 'otherfoo', - } - - # note the effect here is that the first set of params - # takes effect for the rest of them, when one is absent - self.assert_compile( - table.insert().values(values), - 'INSERT INTO sometable (id, data, foo) VALUES ' - '(%(id_0)s, %(data_0)s, %(foo_0)s), ' - '(%(id_1)s, %(data_1)s, %(foo_0)s), ' - '(%(id_2)s, %(data_2)s, %(foo_2)s)', - checkparams=checkparams, - dialect=postgresql.dialect()) + assert_raises_message( + exc.CompileError, + "INSERT value for column sometable.foo is explicitly rendered " + "as a boundparameter in the VALUES clause; a Python-side value or " + "SQL expression is required", + table.insert().values(values).compile + ) -- cgit v1.2.1 From 268bb4d5f6a8c8a23d6f53014980bba58698b4b4 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 13 Jan 2015 15:17:09 -0500 Subject: - refine the previous commit a bit --- lib/sqlalchemy/sql/crud.py | 47 ++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/lib/sqlalchemy/sql/crud.py b/lib/sqlalchemy/sql/crud.py index 4bab69df0..2961f579f 100644 --- a/lib/sqlalchemy/sql/crud.py +++ b/lib/sqlalchemy/sql/crud.py @@ -117,12 +117,11 @@ def _get_crud_params(compiler, stmt, **kw): def _create_bind_param( compiler, col, value, process=True, - required=False, name=None, unique=False): + required=False, name=None): if name is None: name = col.key bindparam = elements.BindParameter( - name, value, - type_=col.type, required=required, unique=unique) + name, value, type_=col.type, required=required) bindparam._is_crud = True if process: bindparam = bindparam._compiler_dispatch(compiler) @@ -301,16 +300,18 @@ def _append_param_insert_pk_returning(compiler, stmt, c, values, kw): ) compiler.returning.append(c) else: - _create_prefetch_bind_param(compiler, c, values) + values.append( + (c, _create_prefetch_bind_param(compiler, c)) + ) + else: compiler.returning.append(c) -def _create_prefetch_bind_param(compiler, c, values, process=True, name=None): - values.append( - (c, _create_bind_param(compiler, c, None, process=process, name=name)) - ) +def _create_prefetch_bind_param(compiler, c, process=True, name=None): + param = _create_bind_param(compiler, c, None, process=process, name=name) compiler.prefetch.append(c) + return param class _multiparam_column(elements.ColumnElement): @@ -325,8 +326,7 @@ class _multiparam_column(elements.ColumnElement): other.original == self.original -def _process_multiparam_default_bind( - compiler, c, index, kw): +def _process_multiparam_default_bind(compiler, c, index, kw): if not c.default: raise exc.CompileError( @@ -337,11 +337,7 @@ def _process_multiparam_default_bind( return compiler.process(c.default.arg.self_group(), **kw) else: col = _multiparam_column(c, index) - bind = _create_bind_param( - compiler, col, None - ) - compiler.prefetch.append(col) - return bind + return _create_prefetch_bind_param(compiler, col) def _append_param_insert_pk(compiler, stmt, c, values, kw): @@ -354,7 +350,9 @@ def _append_param_insert_pk(compiler, stmt, c, values, kw): compiler.dialect. preexecute_autoincrement_sequences) ): - _create_prefetch_bind_param(compiler, c, values) + values.append( + (c, _create_prefetch_bind_param(compiler, c)) + ) def _append_param_insert_hasdefault( @@ -382,7 +380,9 @@ def _append_param_insert_hasdefault( # don't add primary key column to postfetch compiler.postfetch.append(c) else: - _create_prefetch_bind_param(compiler, c, values) + values.append( + (c, _create_prefetch_bind_param(compiler, c)) + ) def _append_param_insert_select_hasdefault( @@ -398,7 +398,9 @@ def _append_param_insert_select_hasdefault( proc = c.default.arg.self_group() values.append((c, proc)) else: - _create_prefetch_bind_param(compiler, c, values, process=False) + values.append( + (c, _create_prefetch_bind_param(compiler, c, process=False)) + ) def _append_param_update( @@ -416,7 +418,9 @@ def _append_param_update( else: compiler.postfetch.append(c) else: - _create_prefetch_bind_param(compiler, c, values) + values.append( + (c, _create_prefetch_bind_param(compiler, c)) + ) elif c.server_onupdate is not None: if implicit_return_defaults and \ c in implicit_return_defaults: @@ -468,7 +472,10 @@ def _get_multitable_params( ) compiler.postfetch.append(c) else: - _create_prefetch_bind_param(compiler, c, values, name=_col_bind_name(c)) + values.append( + (c, _create_prefetch_bind_param( + compiler, c, name=_col_bind_name(c))) + ) elif c.server_onupdate is not None: compiler.postfetch.append(c) -- cgit v1.2.1 From b63aae2c232f980a47aa2a635c35dfa45390f451 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 13 Jan 2015 17:04:35 -0500 Subject: - The "wildcard" loader options, in particular the one set up by the :func:`.orm.load_only` option to cover all attributes not explicitly mentioned, now takes into account the superclasses of a given entity, if that entity is mapped with inheritance mapping, so that attribute names within the superclasses are also omitted from the load. Additionally, the polymorphic discriminator column is unconditionally included in the list, just in the same way that primary key columns are, so that even with load_only() set up, polymorphic loading of subtypes continues to function correctly. fixes #3287 --- doc/build/changelog/changelog_09.rst | 15 ++++ lib/sqlalchemy/orm/mapper.py | 7 ++ lib/sqlalchemy/orm/path_registry.py | 14 ++++ lib/sqlalchemy/orm/strategies.py | 3 +- lib/sqlalchemy/orm/strategy_options.py | 15 +++- test/orm/test_deferred.py | 133 ++++++++++++++++++++++++++++++++- 6 files changed, 181 insertions(+), 6 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index b675be3b0..acead3011 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -14,6 +14,21 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, orm + :versions: 1.0.0 + :tickets: 3287 + + The "wildcard" loader options, in particular the one set up by + the :func:`.orm.load_only` option to cover all attributes not + explicitly mentioned, now takes into account the superclasses + of a given entity, if that entity is mapped with inheritance mapping, + so that attribute names within the superclasses are also omitted + from the load. Additionally, the polymorphic discriminator column + is unconditionally included in the list, just in the same way that + primary key columns are, so that even with load_only() set up, + polymorphic loading of subtypes continues to function correctly. + .. change:: :tags: bug, sql :versions: 1.0.0 diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 0469c2139..74d8f3860 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -2387,6 +2387,13 @@ class Mapper(InspectionAttr): collection.update(self._pks_by_table[table]) return collection + @_memoized_configured_property + def _should_undefer_in_wildcard(self): + cols = set(self.primary_key) + if self.polymorphic_on is not None: + cols.add(self.polymorphic_on) + return cols + @_memoized_configured_property def _primary_key_propkeys(self): return set([prop.key for prop in self._all_pk_props]) diff --git a/lib/sqlalchemy/orm/path_registry.py b/lib/sqlalchemy/orm/path_registry.py index d4dbf29a0..ec80c70cc 100644 --- a/lib/sqlalchemy/orm/path_registry.py +++ b/lib/sqlalchemy/orm/path_registry.py @@ -52,6 +52,9 @@ class PathRegistry(object): """ + is_token = False + is_root = False + def __eq__(self, other): return other is not None and \ self.path == other.path @@ -153,6 +156,8 @@ class RootRegistry(PathRegistry): """ path = () has_entity = False + is_aliased_class = False + is_root = True def __getitem__(self, entity): return entity._path_registry @@ -168,6 +173,15 @@ class TokenRegistry(PathRegistry): has_entity = False + is_token = True + + def generate_for_superclasses(self): + if not self.parent.is_aliased_class and not self.parent.is_root: + for ent in self.parent.mapper.iterate_to_root(): + yield TokenRegistry(self.parent.parent[ent], self.token) + else: + yield self + def __getitem__(self, entity): raise NotImplementedError() diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 81da7ba93..8a4c8e731 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -231,7 +231,8 @@ class DeferredColumnLoader(LoaderStrategy): ( loadopt and 'undefer_pks' in loadopt.local_opts and - set(self.columns).intersection(self.parent.primary_key) + set(self.columns).intersection( + self.parent._should_undefer_in_wildcard) ) or ( diff --git a/lib/sqlalchemy/orm/strategy_options.py b/lib/sqlalchemy/orm/strategy_options.py index 276da2ae0..90e4e9661 100644 --- a/lib/sqlalchemy/orm/strategy_options.py +++ b/lib/sqlalchemy/orm/strategy_options.py @@ -364,6 +364,7 @@ class _UnboundLoad(Load): return None token = start_path[0] + if isinstance(token, util.string_types): entity = self._find_entity_basestring(query, token, raiseerr) elif isinstance(token, PropComparator): @@ -407,10 +408,18 @@ class _UnboundLoad(Load): # prioritize "first class" options over those # that were "links in the chain", e.g. "x" and "y" in # someload("x.y.z") versus someload("x") / someload("x.y") - if self._is_chain_link: - effective_path.setdefault(context, "loader", loader) + + if effective_path.is_token: + for path in effective_path.generate_for_superclasses(): + if self._is_chain_link: + path.setdefault(context, "loader", loader) + else: + path.set(context, "loader", loader) else: - effective_path.set(context, "loader", loader) + if self._is_chain_link: + effective_path.setdefault(context, "loader", loader) + else: + effective_path.set(context, "loader", loader) def _find_entity_prop_comparator(self, query, token, mapper, raiseerr): if _is_aliased_class(mapper): diff --git a/test/orm/test_deferred.py b/test/orm/test_deferred.py index 1457852d8..1b777b527 100644 --- a/test/orm/test_deferred.py +++ b/test/orm/test_deferred.py @@ -2,10 +2,14 @@ import sqlalchemy as sa from sqlalchemy import testing, util from sqlalchemy.orm import mapper, deferred, defer, undefer, Load, \ load_only, undefer_group, create_session, synonym, relationship, Session,\ - joinedload, defaultload + joinedload, defaultload, aliased, contains_eager, with_polymorphic from sqlalchemy.testing import eq_, AssertsCompiledSQL, assert_raises_message from test.orm import _fixtures -from sqlalchemy.orm import strategies + + +from .inheritance._poly_fixtures import Company, Person, Engineer, Manager, \ + Boss, Machine, Paperwork, _Polymorphic + class DeferredTest(AssertsCompiledSQL, _fixtures.FixtureTest): @@ -595,3 +599,128 @@ class DeferredOptionsTest(AssertsCompiledSQL, _fixtures.FixtureTest): ) +class InheritanceTest(_Polymorphic): + __dialect__ = 'default' + + def test_load_only_subclass(self): + s = Session() + q = s.query(Manager).options(load_only("status", "manager_name")) + self.assert_compile( + q, + "SELECT managers.person_id AS managers_person_id, " + "people.person_id AS people_person_id, " + "people.type AS people_type, " + "managers.status AS managers_status, " + "managers.manager_name AS managers_manager_name " + "FROM people JOIN managers " + "ON people.person_id = managers.person_id " + "ORDER BY people.person_id" + ) + + def test_load_only_subclass_and_superclass(self): + s = Session() + q = s.query(Boss).options(load_only("status", "manager_name")) + self.assert_compile( + q, + "SELECT managers.person_id AS managers_person_id, " + "people.person_id AS people_person_id, " + "people.type AS people_type, " + "managers.status AS managers_status, " + "managers.manager_name AS managers_manager_name " + "FROM people JOIN managers " + "ON people.person_id = managers.person_id JOIN boss " + "ON managers.person_id = boss.boss_id ORDER BY people.person_id" + ) + + def test_load_only_alias_subclass(self): + s = Session() + m1 = aliased(Manager, flat=True) + q = s.query(m1).options(load_only("status", "manager_name")) + self.assert_compile( + q, + "SELECT managers_1.person_id AS managers_1_person_id, " + "people_1.person_id AS people_1_person_id, " + "people_1.type AS people_1_type, " + "managers_1.status AS managers_1_status, " + "managers_1.manager_name AS managers_1_manager_name " + "FROM people AS people_1 JOIN managers AS " + "managers_1 ON people_1.person_id = managers_1.person_id " + "ORDER BY people_1.person_id" + ) + + def test_load_only_subclass_from_relationship_polymorphic(self): + s = Session() + wp = with_polymorphic(Person, [Manager], flat=True) + q = s.query(Company).join(Company.employees.of_type(wp)).options( + contains_eager(Company.employees.of_type(wp)). + load_only(wp.Manager.status, wp.Manager.manager_name) + ) + self.assert_compile( + q, + "SELECT people_1.person_id AS people_1_person_id, " + "people_1.type AS people_1_type, " + "managers_1.person_id AS managers_1_person_id, " + "managers_1.status AS managers_1_status, " + "managers_1.manager_name AS managers_1_manager_name, " + "companies.company_id AS companies_company_id, " + "companies.name AS companies_name " + "FROM companies JOIN (people AS people_1 LEFT OUTER JOIN " + "managers AS managers_1 ON people_1.person_id = " + "managers_1.person_id) ON companies.company_id = " + "people_1.company_id" + ) + + def test_load_only_subclass_from_relationship(self): + s = Session() + from sqlalchemy import inspect + inspect(Company).add_property("managers", relationship(Manager)) + q = s.query(Company).join(Company.managers).options( + contains_eager(Company.managers). + load_only("status", "manager_name") + ) + self.assert_compile( + q, + "SELECT companies.company_id AS companies_company_id, " + "companies.name AS companies_name, " + "managers.person_id AS managers_person_id, " + "people.person_id AS people_person_id, " + "people.type AS people_type, " + "managers.status AS managers_status, " + "managers.manager_name AS managers_manager_name " + "FROM companies JOIN (people JOIN managers ON people.person_id = " + "managers.person_id) ON companies.company_id = people.company_id" + ) + + + def test_defer_on_wildcard_subclass(self): + # pretty much the same as load_only except doesn't + # exclude the primary key + + s = Session() + q = s.query(Manager).options( + defer(".*"), undefer("status")) + self.assert_compile( + q, + "SELECT managers.status AS managers_status " + "FROM people JOIN managers ON " + "people.person_id = managers.person_id ORDER BY people.person_id" + ) + + def test_defer_super_name_on_subclass(self): + s = Session() + q = s.query(Manager).options(defer("name")) + self.assert_compile( + q, + "SELECT managers.person_id AS managers_person_id, " + "people.person_id AS people_person_id, " + "people.company_id AS people_company_id, " + "people.type AS people_type, managers.status AS managers_status, " + "managers.manager_name AS managers_manager_name " + "FROM people JOIN managers " + "ON people.person_id = managers.person_id " + "ORDER BY people.person_id" + ) + + + + -- cgit v1.2.1 From 41307cd7339a2a2aee0a3dd9c8b994df99d7eedb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 14 Jan 2015 12:02:41 -0500 Subject: - add new section to ORM referring to runtime inspection API, more links, attempt to fix #3290 --- doc/build/orm/mapping_styles.rst | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/doc/build/orm/mapping_styles.rst b/doc/build/orm/mapping_styles.rst index e6be00ef7..7571ce650 100644 --- a/doc/build/orm/mapping_styles.rst +++ b/doc/build/orm/mapping_styles.rst @@ -119,3 +119,52 @@ systems ultimately create the same configuration, consisting of a :class:`.Table user-defined class, linked together with a :func:`.mapper`. When we talk about "the behavior of :func:`.mapper`", this includes when using the Declarative system as well - it's still used, just behind the scenes. + +Runtime Intropsection of Mappings, Objects +========================================== + +The :class:`.Mapper` object is available from any mapped class, regardless +of method, using the :ref:`core_inspection_toplevel` system. Using the +:func:`.inspect` function, one can acquire the :class:`.Mapper` from a +mapped class:: + + >>> from sqlalchemy import inspect + >>> insp = inspect(User) + +Detailed information is available including :attr:`.Mapper.columns`:: + + >>> insp.columns + + +This is a namespace that can be viewed in a list format or +via individual names:: + + >>> list(insp.columns) + [Column('id', Integer(), table=, primary_key=True, nullable=False), Column('name', String(length=50), table=), Column('fullname', String(length=50), table=), Column('password', String(length=12), table=)] + >>> insp.columns.name + Column('name', String(length=50), table=) + +Other namespaces include :attr:`.Mapper.all_orm_descriptors`, which includes all mapped +attributes as well as hybrids, association proxies:: + + >>> insp.all_orm_descriptors + + >>> insp.all_orm_descriptors.keys() + ['fullname', 'password', 'name', 'id'] + +As well as :attr:`.Mapper.column_attrs`:: + + >>> list(insp.column_attrs) + [, , , ] + >>> insp.column_attrs.name + + >>> insp.column_attrs.name.expression + Column('name', String(length=50), table=) + +.. seealso:: + + :ref:`core_inspection_toplevel` + + :class:`.Mapper` + + :class:`.InstanceState` -- cgit v1.2.1 From 79fa69f1f37fdbc0dfec6bdea1e07f52bfe18f7b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 16 Jan 2015 18:03:45 -0500 Subject: - Fixed bug where Postgresql dialect would fail to render an expression in an :class:`.Index` that did not correspond directly to a table-bound column; typically when a :func:`.text` construct was one of the expressions within the index; or could misinterpret the list of expressions if one or more of them were such an expression. fixes #3174 --- doc/build/changelog/changelog_09.rst | 11 ++++++ lib/sqlalchemy/dialects/postgresql/base.py | 9 +++-- test/dialect/postgresql/test_compiler.py | 54 +++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index acead3011..d9cbd5032 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -14,6 +14,17 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, postgresql + :versions: 1.0.0 + :tickets: 3174 + + Fixed bug where Postgresql dialect would fail to render an + expression in an :class:`.Index` that did not correspond directly + to a table-bound column; typically when a :func:`.text` construct + was one of the expressions within the index; or could misinterpret the + list of expressions if one or more of them were such an expression. + .. change:: :tags: bug, orm :versions: 1.0.0 diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index fa9a2cfd0..0817fe837 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1477,8 +1477,13 @@ class PGDDLCompiler(compiler.DDLCompiler): if not isinstance(expr, expression.ColumnClause) else expr, include_table=False, literal_binds=True) + - (c.key in ops and (' ' + ops[c.key]) or '') - for expr, c in zip(index.expressions, index.columns)]) + ( + (' ' + ops[expr.key]) + if hasattr(expr, 'key') + and expr.key in ops else '' + ) + for expr in index.expressions + ]) ) whereclause = index.dialect_options["postgresql"]["where"] diff --git a/test/dialect/postgresql/test_compiler.py b/test/dialect/postgresql/test_compiler.py index 6c4f3c8cc..5717df9f7 100644 --- a/test/dialect/postgresql/test_compiler.py +++ b/test/dialect/postgresql/test_compiler.py @@ -5,7 +5,7 @@ from sqlalchemy.testing.assertions import AssertsCompiledSQL, is_, \ from sqlalchemy.testing import engines, fixtures from sqlalchemy import testing from sqlalchemy import Sequence, Table, Column, Integer, update, String,\ - insert, func, MetaData, Enum, Index, and_, delete, select, cast + insert, func, MetaData, Enum, Index, and_, delete, select, cast, text from sqlalchemy.dialects.postgresql import ExcludeConstraint, array from sqlalchemy import exc, schema from sqlalchemy.dialects.postgresql import base as postgresql @@ -296,6 +296,58 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): '(data text_pattern_ops, data2 int4_ops)', dialect=postgresql.dialect()) + def test_create_index_with_text_or_composite(self): + m = MetaData() + tbl = Table('testtbl', m, + Column('d1', String), + Column('d2', Integer)) + + idx = Index('test_idx1', text('x')) + tbl.append_constraint(idx) + + idx2 = Index('test_idx2', text('y'), tbl.c.d2) + + idx3 = Index( + 'test_idx2', tbl.c.d1, text('y'), tbl.c.d2, + postgresql_ops={'d1': 'x1', 'd2': 'x2'} + ) + + idx4 = Index( + 'test_idx2', tbl.c.d1, tbl.c.d2 > 5, text('q'), + postgresql_ops={'d1': 'x1', 'd2': 'x2'} + ) + + idx5 = Index( + 'test_idx2', tbl.c.d1, (tbl.c.d2 > 5).label('g'), text('q'), + postgresql_ops={'d1': 'x1', 'g': 'x2'} + ) + + self.assert_compile( + schema.CreateIndex(idx), + "CREATE INDEX test_idx1 ON testtbl (x)" + ) + self.assert_compile( + schema.CreateIndex(idx2), + "CREATE INDEX test_idx2 ON testtbl (y, d2)" + ) + self.assert_compile( + schema.CreateIndex(idx3), + "CREATE INDEX test_idx2 ON testtbl (d1 x1, y, d2 x2)" + ) + + # note that at the moment we do not expect the 'd2' op to + # pick up on the "d2 > 5" expression + self.assert_compile( + schema.CreateIndex(idx4), + "CREATE INDEX test_idx2 ON testtbl (d1 x1, (d2 > 5), q)" + ) + + # however it does work if we label! + self.assert_compile( + schema.CreateIndex(idx5), + "CREATE INDEX test_idx2 ON testtbl (d1 x1, (d2 > 5) x2, q)" + ) + def test_create_index_with_using(self): m = MetaData() tbl = Table('testtbl', m, Column('data', String)) -- cgit v1.2.1 From f3a892a3ef666e299107a990bf4eae7ed9a953ae Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 16 Jan 2015 20:03:33 -0500 Subject: - Custom dialects that implement :class:`.GenericTypeCompiler` can now be constructed such that the visit methods receive an indication of the owning expression object, if any. Any visit method that accepts keyword arguments (e.g. ``**kw``) will in most cases receive a keyword argument ``type_expression``, referring to the expression object that the type is contained within. For columns in DDL, the dialect's compiler class may need to alter its ``get_column_specification()`` method to support this as well. The ``UserDefinedType.get_col_spec()`` method will also receive ``type_expression`` if it provides ``**kw`` in its argument signature. fixes #3074 --- doc/build/changelog/changelog_10.rst | 16 ++++ lib/sqlalchemy/dialects/firebird/base.py | 20 ++--- lib/sqlalchemy/dialects/mssql/base.py | 82 +++++++++--------- lib/sqlalchemy/dialects/mysql/base.py | 76 ++++++++-------- lib/sqlalchemy/dialects/oracle/base.py | 62 ++++++------- lib/sqlalchemy/dialects/postgresql/base.py | 68 +++++++-------- lib/sqlalchemy/dialects/sqlite/base.py | 11 +-- lib/sqlalchemy/dialects/sybase/base.py | 27 +++--- lib/sqlalchemy/sql/compiler.py | 135 +++++++++++++++-------------- lib/sqlalchemy/sql/type_api.py | 24 ++++- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/langhelpers.py | 27 ++++++ test/sql/test_types.py | 63 ++++++++++++++ 13 files changed, 373 insertions(+), 240 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 5d8bb7b68..089c9fafb 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,22 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: enhancement, sql + :tickets: 3074 + + Custom dialects that implement :class:`.GenericTypeCompiler` can + now be constructed such that the visit methods receive an indication + of the owning expression object, if any. Any visit method that + accepts keyword arguments (e.g. ``**kw``) will in most cases + receive a keyword argument ``type_expression``, referring to the + expression object that the type is contained within. For columns + in DDL, the dialect's compiler class may need to alter its + ``get_column_specification()`` method to support this as well. + The ``UserDefinedType.get_col_spec()`` method will also receive + ``type_expression`` if it provides ``**kw`` in its argument + signature. + .. change:: :tags: bug, sql :tickets: 3288 diff --git a/lib/sqlalchemy/dialects/firebird/base.py b/lib/sqlalchemy/dialects/firebird/base.py index 36229a105..74e8abfc2 100644 --- a/lib/sqlalchemy/dialects/firebird/base.py +++ b/lib/sqlalchemy/dialects/firebird/base.py @@ -180,16 +180,16 @@ ischema_names = { # _FBDate, etc. as bind/result functionality is required) class FBTypeCompiler(compiler.GenericTypeCompiler): - def visit_boolean(self, type_): - return self.visit_SMALLINT(type_) + def visit_boolean(self, type_, **kw): + return self.visit_SMALLINT(type_, **kw) - def visit_datetime(self, type_): - return self.visit_TIMESTAMP(type_) + def visit_datetime(self, type_, **kw): + return self.visit_TIMESTAMP(type_, **kw) - def visit_TEXT(self, type_): + def visit_TEXT(self, type_, **kw): return "BLOB SUB_TYPE 1" - def visit_BLOB(self, type_): + def visit_BLOB(self, type_, **kw): return "BLOB SUB_TYPE 0" def _extend_string(self, type_, basic): @@ -199,16 +199,16 @@ class FBTypeCompiler(compiler.GenericTypeCompiler): else: return '%s CHARACTER SET %s' % (basic, charset) - def visit_CHAR(self, type_): - basic = super(FBTypeCompiler, self).visit_CHAR(type_) + def visit_CHAR(self, type_, **kw): + basic = super(FBTypeCompiler, self).visit_CHAR(type_, **kw) return self._extend_string(type_, basic) - def visit_VARCHAR(self, type_): + def visit_VARCHAR(self, type_, **kw): if not type_.length: raise exc.CompileError( "VARCHAR requires a length on dialect %s" % self.dialect.name) - basic = super(FBTypeCompiler, self).visit_VARCHAR(type_) + basic = super(FBTypeCompiler, self).visit_VARCHAR(type_, **kw) return self._extend_string(type_, basic) diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index 5d84975c0..92d7e4ab3 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -694,7 +694,6 @@ ischema_names = { class MSTypeCompiler(compiler.GenericTypeCompiler): - def _extend(self, spec, type_, length=None): """Extend a string-type declaration with standard SQL COLLATE annotations. @@ -715,115 +714,115 @@ class MSTypeCompiler(compiler.GenericTypeCompiler): return ' '.join([c for c in (spec, collation) if c is not None]) - def visit_FLOAT(self, type_): + def visit_FLOAT(self, type_, **kw): precision = getattr(type_, 'precision', None) if precision is None: return "FLOAT" else: return "FLOAT(%(precision)s)" % {'precision': precision} - def visit_TINYINT(self, type_): + def visit_TINYINT(self, type_, **kw): return "TINYINT" - def visit_DATETIMEOFFSET(self, type_): + def visit_DATETIMEOFFSET(self, type_, **kw): if type_.precision: return "DATETIMEOFFSET(%s)" % type_.precision else: return "DATETIMEOFFSET" - def visit_TIME(self, type_): + def visit_TIME(self, type_, **kw): precision = getattr(type_, 'precision', None) if precision: return "TIME(%s)" % precision else: return "TIME" - def visit_DATETIME2(self, type_): + def visit_DATETIME2(self, type_, **kw): precision = getattr(type_, 'precision', None) if precision: return "DATETIME2(%s)" % precision else: return "DATETIME2" - def visit_SMALLDATETIME(self, type_): + def visit_SMALLDATETIME(self, type_, **kw): return "SMALLDATETIME" - def visit_unicode(self, type_): - return self.visit_NVARCHAR(type_) + def visit_unicode(self, type_, **kw): + return self.visit_NVARCHAR(type_, **kw) - def visit_text(self, type_): + def visit_text(self, type_, **kw): if self.dialect.deprecate_large_types: - return self.visit_VARCHAR(type_) + return self.visit_VARCHAR(type_, **kw) else: - return self.visit_TEXT(type_) + return self.visit_TEXT(type_, **kw) - def visit_unicode_text(self, type_): + def visit_unicode_text(self, type_, **kw): if self.dialect.deprecate_large_types: - return self.visit_NVARCHAR(type_) + return self.visit_NVARCHAR(type_, **kw) else: - return self.visit_NTEXT(type_) + return self.visit_NTEXT(type_, **kw) - def visit_NTEXT(self, type_): + def visit_NTEXT(self, type_, **kw): return self._extend("NTEXT", type_) - def visit_TEXT(self, type_): + def visit_TEXT(self, type_, **kw): return self._extend("TEXT", type_) - def visit_VARCHAR(self, type_): + def visit_VARCHAR(self, type_, **kw): return self._extend("VARCHAR", type_, length=type_.length or 'max') - def visit_CHAR(self, type_): + def visit_CHAR(self, type_, **kw): return self._extend("CHAR", type_) - def visit_NCHAR(self, type_): + def visit_NCHAR(self, type_, **kw): return self._extend("NCHAR", type_) - def visit_NVARCHAR(self, type_): + def visit_NVARCHAR(self, type_, **kw): return self._extend("NVARCHAR", type_, length=type_.length or 'max') - def visit_date(self, type_): + def visit_date(self, type_, **kw): if self.dialect.server_version_info < MS_2008_VERSION: - return self.visit_DATETIME(type_) + return self.visit_DATETIME(type_, **kw) else: - return self.visit_DATE(type_) + return self.visit_DATE(type_, **kw) - def visit_time(self, type_): + def visit_time(self, type_, **kw): if self.dialect.server_version_info < MS_2008_VERSION: - return self.visit_DATETIME(type_) + return self.visit_DATETIME(type_, **kw) else: - return self.visit_TIME(type_) + return self.visit_TIME(type_, **kw) - def visit_large_binary(self, type_): + def visit_large_binary(self, type_, **kw): if self.dialect.deprecate_large_types: - return self.visit_VARBINARY(type_) + return self.visit_VARBINARY(type_, **kw) else: - return self.visit_IMAGE(type_) + return self.visit_IMAGE(type_, **kw) - def visit_IMAGE(self, type_): + def visit_IMAGE(self, type_, **kw): return "IMAGE" - def visit_VARBINARY(self, type_): + def visit_VARBINARY(self, type_, **kw): return self._extend( "VARBINARY", type_, length=type_.length or 'max') - def visit_boolean(self, type_): + def visit_boolean(self, type_, **kw): return self.visit_BIT(type_) - def visit_BIT(self, type_): + def visit_BIT(self, type_, **kw): return "BIT" - def visit_MONEY(self, type_): + def visit_MONEY(self, type_, **kw): return "MONEY" - def visit_SMALLMONEY(self, type_): + def visit_SMALLMONEY(self, type_, **kw): return 'SMALLMONEY' - def visit_UNIQUEIDENTIFIER(self, type_): + def visit_UNIQUEIDENTIFIER(self, type_, **kw): return "UNIQUEIDENTIFIER" - def visit_SQL_VARIANT(self, type_): + def visit_SQL_VARIANT(self, type_, **kw): return 'SQL_VARIANT' @@ -1240,8 +1239,11 @@ class MSSQLStrictCompiler(MSSQLCompiler): class MSDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kwargs): - colspec = (self.preparer.format_column(column) + " " - + self.dialect.type_compiler.process(column.type)) + colspec = ( + self.preparer.format_column(column) + " " + + self.dialect.type_compiler.process( + column.type, type_expression=column) + ) if column.nullable is not None: if not column.nullable or column.primary_key or \ diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 9c3f23cb2..ca56a4d23 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -1859,9 +1859,11 @@ class MySQLDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kw): """Builds column DDL.""" - colspec = [self.preparer.format_column(column), - self.dialect.type_compiler.process(column.type) - ] + colspec = [ + self.preparer.format_column(column), + self.dialect.type_compiler.process( + column.type, type_expression=column) + ] default = self.get_column_default_string(column) if default is not None: @@ -2059,7 +2061,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): def _mysql_type(self, type_): return isinstance(type_, (_StringType, _NumericType)) - def visit_NUMERIC(self, type_): + def visit_NUMERIC(self, type_, **kw): if type_.precision is None: return self._extend_numeric(type_, "NUMERIC") elif type_.scale is None: @@ -2072,7 +2074,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): {'precision': type_.precision, 'scale': type_.scale}) - def visit_DECIMAL(self, type_): + def visit_DECIMAL(self, type_, **kw): if type_.precision is None: return self._extend_numeric(type_, "DECIMAL") elif type_.scale is None: @@ -2085,7 +2087,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): {'precision': type_.precision, 'scale': type_.scale}) - def visit_DOUBLE(self, type_): + def visit_DOUBLE(self, type_, **kw): if type_.precision is not None and type_.scale is not None: return self._extend_numeric(type_, "DOUBLE(%(precision)s, %(scale)s)" % @@ -2094,7 +2096,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, 'DOUBLE') - def visit_REAL(self, type_): + def visit_REAL(self, type_, **kw): if type_.precision is not None and type_.scale is not None: return self._extend_numeric(type_, "REAL(%(precision)s, %(scale)s)" % @@ -2103,7 +2105,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, 'REAL') - def visit_FLOAT(self, type_): + def visit_FLOAT(self, type_, **kw): if self._mysql_type(type_) and \ type_.scale is not None and \ type_.precision is not None: @@ -2115,7 +2117,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "FLOAT") - def visit_INTEGER(self, type_): + def visit_INTEGER(self, type_, **kw): if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, "INTEGER(%(display_width)s)" % @@ -2123,7 +2125,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "INTEGER") - def visit_BIGINT(self, type_): + def visit_BIGINT(self, type_, **kw): if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, "BIGINT(%(display_width)s)" % @@ -2131,7 +2133,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "BIGINT") - def visit_MEDIUMINT(self, type_): + def visit_MEDIUMINT(self, type_, **kw): if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, "MEDIUMINT(%(display_width)s)" % @@ -2139,14 +2141,14 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "MEDIUMINT") - def visit_TINYINT(self, type_): + def visit_TINYINT(self, type_, **kw): if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric(type_, "TINYINT(%s)" % type_.display_width) else: return self._extend_numeric(type_, "TINYINT") - def visit_SMALLINT(self, type_): + def visit_SMALLINT(self, type_, **kw): if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric(type_, "SMALLINT(%(display_width)s)" % @@ -2155,55 +2157,55 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "SMALLINT") - def visit_BIT(self, type_): + def visit_BIT(self, type_, **kw): if type_.length is not None: return "BIT(%s)" % type_.length else: return "BIT" - def visit_DATETIME(self, type_): + def visit_DATETIME(self, type_, **kw): if getattr(type_, 'fsp', None): return "DATETIME(%d)" % type_.fsp else: return "DATETIME" - def visit_DATE(self, type_): + def visit_DATE(self, type_, **kw): return "DATE" - def visit_TIME(self, type_): + def visit_TIME(self, type_, **kw): if getattr(type_, 'fsp', None): return "TIME(%d)" % type_.fsp else: return "TIME" - def visit_TIMESTAMP(self, type_): + def visit_TIMESTAMP(self, type_, **kw): if getattr(type_, 'fsp', None): return "TIMESTAMP(%d)" % type_.fsp else: return "TIMESTAMP" - def visit_YEAR(self, type_): + def visit_YEAR(self, type_, **kw): if type_.display_width is None: return "YEAR" else: return "YEAR(%s)" % type_.display_width - def visit_TEXT(self, type_): + def visit_TEXT(self, type_, **kw): if type_.length: return self._extend_string(type_, {}, "TEXT(%d)" % type_.length) else: return self._extend_string(type_, {}, "TEXT") - def visit_TINYTEXT(self, type_): + def visit_TINYTEXT(self, type_, **kw): return self._extend_string(type_, {}, "TINYTEXT") - def visit_MEDIUMTEXT(self, type_): + def visit_MEDIUMTEXT(self, type_, **kw): return self._extend_string(type_, {}, "MEDIUMTEXT") - def visit_LONGTEXT(self, type_): + def visit_LONGTEXT(self, type_, **kw): return self._extend_string(type_, {}, "LONGTEXT") - def visit_VARCHAR(self, type_): + def visit_VARCHAR(self, type_, **kw): if type_.length: return self._extend_string( type_, {}, "VARCHAR(%d)" % type_.length) @@ -2212,14 +2214,14 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): "VARCHAR requires a length on dialect %s" % self.dialect.name) - def visit_CHAR(self, type_): + def visit_CHAR(self, type_, **kw): if type_.length: return self._extend_string(type_, {}, "CHAR(%(length)s)" % {'length': type_.length}) else: return self._extend_string(type_, {}, "CHAR") - def visit_NVARCHAR(self, type_): + def visit_NVARCHAR(self, type_, **kw): # We'll actually generate the equiv. "NATIONAL VARCHAR" instead # of "NVARCHAR". if type_.length: @@ -2231,7 +2233,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): "NVARCHAR requires a length on dialect %s" % self.dialect.name) - def visit_NCHAR(self, type_): + def visit_NCHAR(self, type_, **kw): # We'll actually generate the equiv. # "NATIONAL CHAR" instead of "NCHAR". if type_.length: @@ -2241,31 +2243,31 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_string(type_, {'national': True}, "CHAR") - def visit_VARBINARY(self, type_): + def visit_VARBINARY(self, type_, **kw): return "VARBINARY(%d)" % type_.length - def visit_large_binary(self, type_): + def visit_large_binary(self, type_, **kw): return self.visit_BLOB(type_) - def visit_enum(self, type_): + def visit_enum(self, type_, **kw): if not type_.native_enum: return super(MySQLTypeCompiler, self).visit_enum(type_) else: return self._visit_enumerated_values("ENUM", type_, type_.enums) - def visit_BLOB(self, type_): + def visit_BLOB(self, type_, **kw): if type_.length: return "BLOB(%d)" % type_.length else: return "BLOB" - def visit_TINYBLOB(self, type_): + def visit_TINYBLOB(self, type_, **kw): return "TINYBLOB" - def visit_MEDIUMBLOB(self, type_): + def visit_MEDIUMBLOB(self, type_, **kw): return "MEDIUMBLOB" - def visit_LONGBLOB(self, type_): + def visit_LONGBLOB(self, type_, **kw): return "LONGBLOB" def _visit_enumerated_values(self, name, type_, enumerated_values): @@ -2276,15 +2278,15 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): name, ",".join(quoted_enums)) ) - def visit_ENUM(self, type_): + def visit_ENUM(self, type_, **kw): return self._visit_enumerated_values("ENUM", type_, type_._enumerated_values) - def visit_SET(self, type_): + def visit_SET(self, type_, **kw): return self._visit_enumerated_values("SET", type_, type_._enumerated_values) - def visit_BOOLEAN(self, type): + def visit_BOOLEAN(self, type, **kw): return "BOOL" diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py index 9f375da94..b482c9069 100644 --- a/lib/sqlalchemy/dialects/oracle/base.py +++ b/lib/sqlalchemy/dialects/oracle/base.py @@ -457,19 +457,19 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): # Oracle does not allow milliseconds in DATE # Oracle does not support TIME columns - def visit_datetime(self, type_): - return self.visit_DATE(type_) + def visit_datetime(self, type_, **kw): + return self.visit_DATE(type_, **kw) - def visit_float(self, type_): - return self.visit_FLOAT(type_) + def visit_float(self, type_, **kw): + return self.visit_FLOAT(type_, **kw) - def visit_unicode(self, type_): + def visit_unicode(self, type_, **kw): if self.dialect._supports_nchar: - return self.visit_NVARCHAR2(type_) + return self.visit_NVARCHAR2(type_, **kw) else: - return self.visit_VARCHAR2(type_) + return self.visit_VARCHAR2(type_, **kw) - def visit_INTERVAL(self, type_): + def visit_INTERVAL(self, type_, **kw): return "INTERVAL DAY%s TO SECOND%s" % ( type_.day_precision is not None and "(%d)" % type_.day_precision or @@ -479,22 +479,22 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): "", ) - def visit_LONG(self, type_): + def visit_LONG(self, type_, **kw): return "LONG" - def visit_TIMESTAMP(self, type_): + def visit_TIMESTAMP(self, type_, **kw): if type_.timezone: return "TIMESTAMP WITH TIME ZONE" else: return "TIMESTAMP" - def visit_DOUBLE_PRECISION(self, type_): - return self._generate_numeric(type_, "DOUBLE PRECISION") + def visit_DOUBLE_PRECISION(self, type_, **kw): + return self._generate_numeric(type_, "DOUBLE PRECISION", **kw) def visit_NUMBER(self, type_, **kw): return self._generate_numeric(type_, "NUMBER", **kw) - def _generate_numeric(self, type_, name, precision=None, scale=None): + def _generate_numeric(self, type_, name, precision=None, scale=None, **kw): if precision is None: precision = type_.precision @@ -510,17 +510,17 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): n = "%(name)s(%(precision)s, %(scale)s)" return n % {'name': name, 'precision': precision, 'scale': scale} - def visit_string(self, type_): - return self.visit_VARCHAR2(type_) + def visit_string(self, type_, **kw): + return self.visit_VARCHAR2(type_, **kw) - def visit_VARCHAR2(self, type_): + def visit_VARCHAR2(self, type_, **kw): return self._visit_varchar(type_, '', '2') - def visit_NVARCHAR2(self, type_): + def visit_NVARCHAR2(self, type_, **kw): return self._visit_varchar(type_, 'N', '2') visit_NVARCHAR = visit_NVARCHAR2 - def visit_VARCHAR(self, type_): + def visit_VARCHAR(self, type_, **kw): return self._visit_varchar(type_, '', '') def _visit_varchar(self, type_, n, num): @@ -533,31 +533,31 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): varchar = "%(n)sVARCHAR%(two)s(%(length)s)" return varchar % {'length': type_.length, 'two': num, 'n': n} - def visit_text(self, type_): - return self.visit_CLOB(type_) + def visit_text(self, type_, **kw): + return self.visit_CLOB(type_, **kw) - def visit_unicode_text(self, type_): + def visit_unicode_text(self, type_, **kw): if self.dialect._supports_nchar: - return self.visit_NCLOB(type_) + return self.visit_NCLOB(type_, **kw) else: - return self.visit_CLOB(type_) + return self.visit_CLOB(type_, **kw) - def visit_large_binary(self, type_): - return self.visit_BLOB(type_) + def visit_large_binary(self, type_, **kw): + return self.visit_BLOB(type_, **kw) - def visit_big_integer(self, type_): - return self.visit_NUMBER(type_, precision=19) + def visit_big_integer(self, type_, **kw): + return self.visit_NUMBER(type_, precision=19, **kw) - def visit_boolean(self, type_): - return self.visit_SMALLINT(type_) + def visit_boolean(self, type_, **kw): + return self.visit_SMALLINT(type_, **kw) - def visit_RAW(self, type_): + def visit_RAW(self, type_, **kw): if type_.length: return "RAW(%(length)s)" % {'length': type_.length} else: return "RAW" - def visit_ROWID(self, type_): + def visit_ROWID(self, type_, **kw): return "ROWID" diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 0817fe837..89bea100e 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1425,7 +1425,8 @@ class PGDDLCompiler(compiler.DDLCompiler): else: colspec += " SERIAL" else: - colspec += " " + self.dialect.type_compiler.process(column.type) + colspec += " " + self.dialect.type_compiler.process(column.type, + type_expression=column) default = self.get_column_default_string(column) if default is not None: colspec += " DEFAULT " + default @@ -1545,94 +1546,93 @@ class PGDDLCompiler(compiler.DDLCompiler): class PGTypeCompiler(compiler.GenericTypeCompiler): - - def visit_TSVECTOR(self, type): + def visit_TSVECTOR(self, type, **kw): return "TSVECTOR" - def visit_INET(self, type_): + def visit_INET(self, type_, **kw): return "INET" - def visit_CIDR(self, type_): + def visit_CIDR(self, type_, **kw): return "CIDR" - def visit_MACADDR(self, type_): + def visit_MACADDR(self, type_, **kw): return "MACADDR" - def visit_OID(self, type_): + def visit_OID(self, type_, **kw): return "OID" - def visit_FLOAT(self, type_): + def visit_FLOAT(self, type_, **kw): if not type_.precision: return "FLOAT" else: return "FLOAT(%(precision)s)" % {'precision': type_.precision} - def visit_DOUBLE_PRECISION(self, type_): + def visit_DOUBLE_PRECISION(self, type_, **kw): return "DOUBLE PRECISION" - def visit_BIGINT(self, type_): + def visit_BIGINT(self, type_, **kw): return "BIGINT" - def visit_HSTORE(self, type_): + def visit_HSTORE(self, type_, **kw): return "HSTORE" - def visit_JSON(self, type_): + def visit_JSON(self, type_, **kw): return "JSON" - def visit_JSONB(self, type_): + def visit_JSONB(self, type_, **kw): return "JSONB" - def visit_INT4RANGE(self, type_): + def visit_INT4RANGE(self, type_, **kw): return "INT4RANGE" - def visit_INT8RANGE(self, type_): + def visit_INT8RANGE(self, type_, **kw): return "INT8RANGE" - def visit_NUMRANGE(self, type_): + def visit_NUMRANGE(self, type_, **kw): return "NUMRANGE" - def visit_DATERANGE(self, type_): + def visit_DATERANGE(self, type_, **kw): return "DATERANGE" - def visit_TSRANGE(self, type_): + def visit_TSRANGE(self, type_, **kw): return "TSRANGE" - def visit_TSTZRANGE(self, type_): + def visit_TSTZRANGE(self, type_, **kw): return "TSTZRANGE" - def visit_datetime(self, type_): - return self.visit_TIMESTAMP(type_) + def visit_datetime(self, type_, **kw): + return self.visit_TIMESTAMP(type_, **kw) - def visit_enum(self, type_): + def visit_enum(self, type_, **kw): if not type_.native_enum or not self.dialect.supports_native_enum: - return super(PGTypeCompiler, self).visit_enum(type_) + return super(PGTypeCompiler, self).visit_enum(type_, **kw) else: - return self.visit_ENUM(type_) + return self.visit_ENUM(type_, **kw) - def visit_ENUM(self, type_): + def visit_ENUM(self, type_, **kw): return self.dialect.identifier_preparer.format_type(type_) - def visit_TIMESTAMP(self, type_): + def visit_TIMESTAMP(self, type_, **kw): return "TIMESTAMP%s %s" % ( getattr(type_, 'precision', None) and "(%d)" % type_.precision or "", (type_.timezone and "WITH" or "WITHOUT") + " TIME ZONE" ) - def visit_TIME(self, type_): + def visit_TIME(self, type_, **kw): return "TIME%s %s" % ( getattr(type_, 'precision', None) and "(%d)" % type_.precision or "", (type_.timezone and "WITH" or "WITHOUT") + " TIME ZONE" ) - def visit_INTERVAL(self, type_): + def visit_INTERVAL(self, type_, **kw): if type_.precision is not None: return "INTERVAL(%d)" % type_.precision else: return "INTERVAL" - def visit_BIT(self, type_): + def visit_BIT(self, type_, **kw): if type_.varying: compiled = "BIT VARYING" if type_.length is not None: @@ -1641,16 +1641,16 @@ class PGTypeCompiler(compiler.GenericTypeCompiler): compiled = "BIT(%d)" % type_.length return compiled - def visit_UUID(self, type_): + def visit_UUID(self, type_, **kw): return "UUID" - def visit_large_binary(self, type_): - return self.visit_BYTEA(type_) + def visit_large_binary(self, type_, **kw): + return self.visit_BYTEA(type_, **kw) - def visit_BYTEA(self, type_): + def visit_BYTEA(self, type_, **kw): return "BYTEA" - def visit_ARRAY(self, type_): + def visit_ARRAY(self, type_, **kw): return self.process(type_.item_type) + ('[]' * (type_.dimensions if type_.dimensions is not None else 1)) diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index 3d7b0788b..f74421967 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -660,7 +660,8 @@ class SQLiteCompiler(compiler.SQLCompiler): class SQLiteDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kwargs): - coltype = self.dialect.type_compiler.process(column.type) + coltype = self.dialect.type_compiler.process( + column.type, type_expression=column) colspec = self.preparer.format_column(column) + " " + coltype default = self.get_column_default_string(column) if default is not None: @@ -716,24 +717,24 @@ class SQLiteDDLCompiler(compiler.DDLCompiler): class SQLiteTypeCompiler(compiler.GenericTypeCompiler): - def visit_large_binary(self, type_): + def visit_large_binary(self, type_, **kw): return self.visit_BLOB(type_) - def visit_DATETIME(self, type_): + def visit_DATETIME(self, type_, **kw): if not isinstance(type_, _DateTimeMixin) or \ type_.format_is_text_affinity: return super(SQLiteTypeCompiler, self).visit_DATETIME(type_) else: return "DATETIME_CHAR" - def visit_DATE(self, type_): + def visit_DATE(self, type_, **kw): if not isinstance(type_, _DateTimeMixin) or \ type_.format_is_text_affinity: return super(SQLiteTypeCompiler, self).visit_DATE(type_) else: return "DATE_CHAR" - def visit_TIME(self, type_): + def visit_TIME(self, type_, **kw): if not isinstance(type_, _DateTimeMixin) or \ type_.format_is_text_affinity: return super(SQLiteTypeCompiler, self).visit_TIME(type_) diff --git a/lib/sqlalchemy/dialects/sybase/base.py b/lib/sqlalchemy/dialects/sybase/base.py index f65a76a27..369420358 100644 --- a/lib/sqlalchemy/dialects/sybase/base.py +++ b/lib/sqlalchemy/dialects/sybase/base.py @@ -146,40 +146,40 @@ class IMAGE(sqltypes.LargeBinary): class SybaseTypeCompiler(compiler.GenericTypeCompiler): - def visit_large_binary(self, type_): + def visit_large_binary(self, type_, **kw): return self.visit_IMAGE(type_) - def visit_boolean(self, type_): + def visit_boolean(self, type_, **kw): return self.visit_BIT(type_) - def visit_unicode(self, type_): + def visit_unicode(self, type_, **kw): return self.visit_NVARCHAR(type_) - def visit_UNICHAR(self, type_): + def visit_UNICHAR(self, type_, **kw): return "UNICHAR(%d)" % type_.length - def visit_UNIVARCHAR(self, type_): + def visit_UNIVARCHAR(self, type_, **kw): return "UNIVARCHAR(%d)" % type_.length - def visit_UNITEXT(self, type_): + def visit_UNITEXT(self, type_, **kw): return "UNITEXT" - def visit_TINYINT(self, type_): + def visit_TINYINT(self, type_, **kw): return "TINYINT" - def visit_IMAGE(self, type_): + def visit_IMAGE(self, type_, **kw): return "IMAGE" - def visit_BIT(self, type_): + def visit_BIT(self, type_, **kw): return "BIT" - def visit_MONEY(self, type_): + def visit_MONEY(self, type_, **kw): return "MONEY" - def visit_SMALLMONEY(self, type_): + def visit_SMALLMONEY(self, type_, **kw): return "SMALLMONEY" - def visit_UNIQUEIDENTIFIER(self, type_): + def visit_UNIQUEIDENTIFIER(self, type_, **kw): return "UNIQUEIDENTIFIER" ischema_names = { @@ -377,7 +377,8 @@ class SybaseSQLCompiler(compiler.SQLCompiler): class SybaseDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kwargs): colspec = self.preparer.format_column(column) + " " + \ - self.dialect.type_compiler.process(column.type) + self.dialect.type_compiler.process( + column.type, type_expression=column) if column.table is None: raise exc.CompileError( diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index ca14c9371..da62b1434 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -248,15 +248,16 @@ class Compiled(object): return self.execute(*multiparams, **params).scalar() -class TypeCompiler(object): - +class TypeCompiler(util.with_metaclass(util.EnsureKWArgType, object)): """Produces DDL specification for TypeEngine objects.""" + ensure_kwarg = 'visit_\w+' + def __init__(self, dialect): self.dialect = dialect - def process(self, type_): - return type_._compiler_dispatch(self) + def process(self, type_, **kw): + return type_._compiler_dispatch(self, **kw) class _CompileLabel(visitors.Visitable): @@ -638,8 +639,9 @@ class SQLCompiler(Compiled): def visit_index(self, index, **kwargs): return index.name - def visit_typeclause(self, typeclause, **kwargs): - return self.dialect.type_compiler.process(typeclause.type) + def visit_typeclause(self, typeclause, **kw): + kw['type_expression'] = typeclause + return self.dialect.type_compiler.process(typeclause.type, **kw) def post_process_text(self, text): return text @@ -2259,7 +2261,8 @@ class DDLCompiler(Compiled): def get_column_specification(self, column, **kwargs): colspec = self.preparer.format_column(column) + " " + \ - self.dialect.type_compiler.process(column.type) + self.dialect.type_compiler.process( + column.type, type_expression=column) default = self.get_column_default_string(column) if default is not None: colspec += " DEFAULT " + default @@ -2383,13 +2386,13 @@ class DDLCompiler(Compiled): class GenericTypeCompiler(TypeCompiler): - def visit_FLOAT(self, type_): + def visit_FLOAT(self, type_, **kw): return "FLOAT" - def visit_REAL(self, type_): + def visit_REAL(self, type_, **kw): return "REAL" - def visit_NUMERIC(self, type_): + def visit_NUMERIC(self, type_, **kw): if type_.precision is None: return "NUMERIC" elif type_.scale is None: @@ -2400,7 +2403,7 @@ class GenericTypeCompiler(TypeCompiler): {'precision': type_.precision, 'scale': type_.scale} - def visit_DECIMAL(self, type_): + def visit_DECIMAL(self, type_, **kw): if type_.precision is None: return "DECIMAL" elif type_.scale is None: @@ -2411,31 +2414,31 @@ class GenericTypeCompiler(TypeCompiler): {'precision': type_.precision, 'scale': type_.scale} - def visit_INTEGER(self, type_): + def visit_INTEGER(self, type_, **kw): return "INTEGER" - def visit_SMALLINT(self, type_): + def visit_SMALLINT(self, type_, **kw): return "SMALLINT" - def visit_BIGINT(self, type_): + def visit_BIGINT(self, type_, **kw): return "BIGINT" - def visit_TIMESTAMP(self, type_): + def visit_TIMESTAMP(self, type_, **kw): return 'TIMESTAMP' - def visit_DATETIME(self, type_): + def visit_DATETIME(self, type_, **kw): return "DATETIME" - def visit_DATE(self, type_): + def visit_DATE(self, type_, **kw): return "DATE" - def visit_TIME(self, type_): + def visit_TIME(self, type_, **kw): return "TIME" - def visit_CLOB(self, type_): + def visit_CLOB(self, type_, **kw): return "CLOB" - def visit_NCLOB(self, type_): + def visit_NCLOB(self, type_, **kw): return "NCLOB" def _render_string_type(self, type_, name): @@ -2447,91 +2450,91 @@ class GenericTypeCompiler(TypeCompiler): text += ' COLLATE "%s"' % type_.collation return text - def visit_CHAR(self, type_): + def visit_CHAR(self, type_, **kw): return self._render_string_type(type_, "CHAR") - def visit_NCHAR(self, type_): + def visit_NCHAR(self, type_, **kw): return self._render_string_type(type_, "NCHAR") - def visit_VARCHAR(self, type_): + def visit_VARCHAR(self, type_, **kw): return self._render_string_type(type_, "VARCHAR") - def visit_NVARCHAR(self, type_): + def visit_NVARCHAR(self, type_, **kw): return self._render_string_type(type_, "NVARCHAR") - def visit_TEXT(self, type_): + def visit_TEXT(self, type_, **kw): return self._render_string_type(type_, "TEXT") - def visit_BLOB(self, type_): + def visit_BLOB(self, type_, **kw): return "BLOB" - def visit_BINARY(self, type_): + def visit_BINARY(self, type_, **kw): return "BINARY" + (type_.length and "(%d)" % type_.length or "") - def visit_VARBINARY(self, type_): + def visit_VARBINARY(self, type_, **kw): return "VARBINARY" + (type_.length and "(%d)" % type_.length or "") - def visit_BOOLEAN(self, type_): + def visit_BOOLEAN(self, type_, **kw): return "BOOLEAN" - def visit_large_binary(self, type_): - return self.visit_BLOB(type_) + def visit_large_binary(self, type_, **kw): + return self.visit_BLOB(type_, **kw) - def visit_boolean(self, type_): - return self.visit_BOOLEAN(type_) + def visit_boolean(self, type_, **kw): + return self.visit_BOOLEAN(type_, **kw) - def visit_time(self, type_): - return self.visit_TIME(type_) + def visit_time(self, type_, **kw): + return self.visit_TIME(type_, **kw) - def visit_datetime(self, type_): - return self.visit_DATETIME(type_) + def visit_datetime(self, type_, **kw): + return self.visit_DATETIME(type_, **kw) - def visit_date(self, type_): - return self.visit_DATE(type_) + def visit_date(self, type_, **kw): + return self.visit_DATE(type_, **kw) - def visit_big_integer(self, type_): - return self.visit_BIGINT(type_) + def visit_big_integer(self, type_, **kw): + return self.visit_BIGINT(type_, **kw) - def visit_small_integer(self, type_): - return self.visit_SMALLINT(type_) + def visit_small_integer(self, type_, **kw): + return self.visit_SMALLINT(type_, **kw) - def visit_integer(self, type_): - return self.visit_INTEGER(type_) + def visit_integer(self, type_, **kw): + return self.visit_INTEGER(type_, **kw) - def visit_real(self, type_): - return self.visit_REAL(type_) + def visit_real(self, type_, **kw): + return self.visit_REAL(type_, **kw) - def visit_float(self, type_): - return self.visit_FLOAT(type_) + def visit_float(self, type_, **kw): + return self.visit_FLOAT(type_, **kw) - def visit_numeric(self, type_): - return self.visit_NUMERIC(type_) + def visit_numeric(self, type_, **kw): + return self.visit_NUMERIC(type_, **kw) - def visit_string(self, type_): - return self.visit_VARCHAR(type_) + def visit_string(self, type_, **kw): + return self.visit_VARCHAR(type_, **kw) - def visit_unicode(self, type_): - return self.visit_VARCHAR(type_) + def visit_unicode(self, type_, **kw): + return self.visit_VARCHAR(type_, **kw) - def visit_text(self, type_): - return self.visit_TEXT(type_) + def visit_text(self, type_, **kw): + return self.visit_TEXT(type_, **kw) - def visit_unicode_text(self, type_): - return self.visit_TEXT(type_) + def visit_unicode_text(self, type_, **kw): + return self.visit_TEXT(type_, **kw) - def visit_enum(self, type_): - return self.visit_VARCHAR(type_) + def visit_enum(self, type_, **kw): + return self.visit_VARCHAR(type_, **kw) - def visit_null(self, type_): + def visit_null(self, type_, **kw): raise exc.CompileError("Can't generate DDL for %r; " "did you forget to specify a " "type on this Column?" % type_) - def visit_type_decorator(self, type_): - return self.process(type_.type_engine(self.dialect)) + def visit_type_decorator(self, type_, **kw): + return self.process(type_.type_engine(self.dialect), **kw) - def visit_user_defined(self, type_): - return type_.get_col_spec() + def visit_user_defined(self, type_, **kw): + return type_.get_col_spec(**kw) class IdentifierPreparer(object): diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py index bff497800..19398ae96 100644 --- a/lib/sqlalchemy/sql/type_api.py +++ b/lib/sqlalchemy/sql/type_api.py @@ -12,7 +12,7 @@ from .. import exc, util from . import operators -from .visitors import Visitable +from .visitors import Visitable, VisitableType # these are back-assigned by sqltypes. BOOLEANTYPE = None @@ -460,7 +460,11 @@ class TypeEngine(Visitable): return util.generic_repr(self) -class UserDefinedType(TypeEngine): +class VisitableCheckKWArg(util.EnsureKWArgType, VisitableType): + pass + + +class UserDefinedType(util.with_metaclass(VisitableCheckKWArg, TypeEngine)): """Base for user defined types. This should be the base of new types. Note that @@ -473,7 +477,7 @@ class UserDefinedType(TypeEngine): def __init__(self, precision = 8): self.precision = precision - def get_col_spec(self): + def get_col_spec(self, **kw): return "MYTYPE(%s)" % self.precision def bind_processor(self, dialect): @@ -493,9 +497,23 @@ class UserDefinedType(TypeEngine): Column('data', MyType(16)) ) + The ``get_col_spec()`` method will in most cases receive a keyword + argument ``type_expression`` which refers to the owning expression + of the type as being compiled, such as a :class:`.Column` or + :func:`.cast` construct. This keyword is only sent if the method + accepts keyword arguments (e.g. ``**kw``) in its argument signature; + introspection is used to check for this in order to support legacy + forms of this function. + + .. versionadded:: 1.0.0 the owning expression is passed to + the ``get_col_spec()`` method via the keyword argument + ``type_expression``, if it receives ``**kw`` in its signature. + """ __visit_name__ = "user_defined" + ensure_kwarg = 'get_col_spec' + class Comparator(TypeEngine.Comparator): __slots__ = () diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index c23b0196f..ceee18d86 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -36,7 +36,7 @@ from .langhelpers import iterate_attributes, class_hierarchy, \ generic_repr, counter, PluginLoader, hybridproperty, hybridmethod, \ safe_reraise,\ get_callable_argspec, only_once, attrsetter, ellipses_string, \ - warn_limited, map_bits, MemoizedSlots + warn_limited, map_bits, MemoizedSlots, EnsureKWArgType from .deprecations import warn_deprecated, warn_pending_deprecation, \ deprecated, pending_deprecation, inject_docstring_text diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 22b6ad4ca..5a938501a 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -1348,6 +1348,7 @@ def chop_traceback(tb, exclude_prefix=_UNITTEST_RE, exclude_suffix=_SQLA_RE): NoneType = type(None) + def attrsetter(attrname): code = \ "def set(obj, value):"\ @@ -1355,3 +1356,29 @@ def attrsetter(attrname): env = locals().copy() exec(code, env) return env['set'] + + +class EnsureKWArgType(type): + """Apply translation of functions to accept **kw arguments if they + don't already. + + """ + def __init__(cls, clsname, bases, clsdict): + fn_reg = cls.ensure_kwarg + if fn_reg: + for key in clsdict: + m = re.match(fn_reg, key) + if m: + fn = clsdict[key] + spec = inspect.getargspec(fn) + if not spec.keywords: + clsdict[key] = wrapped = cls._wrap_w_kw(fn) + setattr(cls, key, wrapped) + super(EnsureKWArgType, cls).__init__(clsname, bases, clsdict) + + def _wrap_w_kw(self, fn): + + def wrap(*arg, **kw): + return fn(*arg) + return update_wrapper(wrap, fn) + diff --git a/test/sql/test_types.py b/test/sql/test_types.py index 6ffd88d78..38b3ced13 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -10,6 +10,8 @@ from sqlalchemy import ( type_coerce, VARCHAR, Time, DateTime, BigInteger, SmallInteger, BOOLEAN, BLOB, NCHAR, NVARCHAR, CLOB, TIME, DATE, DATETIME, TIMESTAMP, SMALLINT, INTEGER, DECIMAL, NUMERIC, FLOAT, REAL) +from sqlalchemy.sql import ddl + from sqlalchemy import exc, types, util, dialects for name in dialects.__all__: __import__("sqlalchemy.dialects.%s" % name) @@ -309,6 +311,24 @@ class UserDefinedTest(fixtures.TablesTest, AssertsCompiledSQL): literal_binds=True ) + def test_kw_colspec(self): + class MyType(types.UserDefinedType): + def get_col_spec(self, **kw): + return "FOOB %s" % kw['type_expression'].name + + class MyOtherType(types.UserDefinedType): + def get_col_spec(self): + return "BAR" + + self.assert_compile( + ddl.CreateColumn(Column('bar', MyType)), + "bar FOOB bar" + ) + self.assert_compile( + ddl.CreateColumn(Column('bar', MyOtherType)), + "bar BAR" + ) + def test_typedecorator_literal_render_fallback_bound(self): # fall back to process_bind_param for literal # value rendering. @@ -1642,6 +1662,49 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): def test_decimal_scale(self): self.assert_compile(types.DECIMAL(2, 4), 'DECIMAL(2, 4)') + def test_kwarg_legacy_typecompiler(self): + from sqlalchemy.sql import compiler + + class SomeTypeCompiler(compiler.GenericTypeCompiler): + # transparently decorated w/ kw decorator + def visit_VARCHAR(self, type_): + return "MYVARCHAR" + + # not affected + def visit_INTEGER(self, type_, **kw): + return "MYINTEGER %s" % kw['type_expression'].name + + dialect = default.DefaultDialect() + dialect.type_compiler = SomeTypeCompiler(dialect) + self.assert_compile( + ddl.CreateColumn(Column('bar', VARCHAR(50))), + "bar MYVARCHAR", + dialect=dialect + ) + self.assert_compile( + ddl.CreateColumn(Column('bar', INTEGER)), + "bar MYINTEGER bar", + dialect=dialect + ) + + +class TestKWArgPassThru(AssertsCompiledSQL, fixtures.TestBase): + __backend__ = True + + def test_user_defined(self): + """test that dialects pass the column through on DDL.""" + + class MyType(types.UserDefinedType): + def get_col_spec(self, **kw): + return "FOOB %s" % kw['type_expression'].name + + m = MetaData() + t = Table('t', m, Column('bar', MyType)) + self.assert_compile( + ddl.CreateColumn(t.c.bar), + "bar FOOB bar" + ) + class NumericRawSQLTest(fixtures.TestBase): -- cgit v1.2.1 From 469b6fabaf78fa0aad485005fd7bc8be7fe27f92 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 17 Jan 2015 12:46:20 -0500 Subject: - add an exclusion here that helps with the case of 3rd party test suite redefining an existing test in test_suite --- lib/sqlalchemy/testing/plugin/pytestplugin.py | 3 ++- test/dialect/test_suite.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index 4bbc8ed9a..fbab4966c 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -84,7 +84,8 @@ def pytest_collection_modifyitems(session, config, items): rebuilt_items = collections.defaultdict(list) items[:] = [ item for item in - items if isinstance(item.parent, pytest.Instance)] + items if isinstance(item.parent, pytest.Instance) + and not item.parent.parent.name.startswith("_")] test_classes = set(item.parent for item in items) for test_class in test_classes: for sub_cls in plugin_base.generate_sub_tests( diff --git a/test/dialect/test_suite.py b/test/dialect/test_suite.py index e6d642ced..3820a7721 100644 --- a/test/dialect/test_suite.py +++ b/test/dialect/test_suite.py @@ -1,2 +1,3 @@ from sqlalchemy.testing.suite import * + -- cgit v1.2.1 From f49c367ef712d080e630ba722f96903922d7de7b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 17 Jan 2015 21:36:52 -0500 Subject: - fix a regression from ref #3178, where dialects that don't actually support sane multi rowcount (e.g. pyodbc) would fail on multirow update. add a test that mocks this breakage into plain dialects --- lib/sqlalchemy/orm/persistence.py | 16 ++++++--- test/orm/test_unitofworkv2.py | 68 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index f477e1dd7..dbf1d3eb4 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -617,6 +617,14 @@ def _emit_update_statements(base_mapper, uowtransaction, rows = 0 records = list(records) + # TODO: would be super-nice to not have to determine this boolean + # inside the loop here, in the 99.9999% of the time there's only + # one connection in use + assert_singlerow = connection.dialect.supports_sane_rowcount + assert_multirow = assert_singlerow and \ + connection.dialect.supports_sane_multi_rowcount + allow_multirow = not needs_version_id or assert_multirow + if hasvalue: for state, state_dict, params, mapper, \ connection, value_params in records: @@ -635,9 +643,7 @@ def _emit_update_statements(base_mapper, uowtransaction, value_params) rows += c.rowcount else: - if needs_version_id and \ - not connection.dialect.supports_sane_multi_rowcount and \ - connection.dialect.supports_sane_rowcount: + if not allow_multirow: for state, state_dict, params, mapper, \ connection, value_params in records: c = cached_connections[connection].\ @@ -654,6 +660,7 @@ def _emit_update_statements(base_mapper, uowtransaction, rows += c.rowcount else: multiparams = [rec[2] for rec in records] + c = cached_connections[connection].\ execute(statement, multiparams) @@ -670,7 +677,8 @@ def _emit_update_statements(base_mapper, uowtransaction, c.context.compiled_parameters[0], value_params) - if connection.dialect.supports_sane_rowcount: + if assert_multirow or assert_singlerow and \ + len(multiparams) == 1: if rows != len(records): raise orm_exc.StaleDataError( "UPDATE statement on table '%s' expected to " diff --git a/test/orm/test_unitofworkv2.py b/test/orm/test_unitofworkv2.py index 374a77237..681b104cf 100644 --- a/test/orm/test_unitofworkv2.py +++ b/test/orm/test_unitofworkv2.py @@ -3,13 +3,13 @@ from sqlalchemy import testing from sqlalchemy.testing import engines from sqlalchemy.testing.schema import Table, Column from test.orm import _fixtures -from sqlalchemy import exc -from sqlalchemy.testing import fixtures +from sqlalchemy import exc, util +from sqlalchemy.testing import fixtures, config from sqlalchemy import Integer, String, ForeignKey, func from sqlalchemy.orm import mapper, relationship, backref, \ create_session, unitofwork, attributes,\ Session, exc as orm_exc -from sqlalchemy.testing.mock import Mock +from sqlalchemy.testing.mock import Mock, patch from sqlalchemy.testing.assertsql import AllOf, CompiledSQL from sqlalchemy import event @@ -1473,6 +1473,67 @@ class BasicStaleChecksTest(fixtures.MappedTest): sess.flush ) + def test_update_single_missing_broken_multi_rowcount(self): + @util.memoized_property + def rowcount(self): + if len(self.context.compiled_parameters) > 1: + return -1 + else: + return self.context.rowcount + + with patch.object( + config.db.dialect, "supports_sane_multi_rowcount", False): + with patch( + "sqlalchemy.engine.result.ResultProxy.rowcount", + rowcount): + Parent, Child = self._fixture() + sess = Session() + p1 = Parent(id=1, data=2) + sess.add(p1) + sess.flush() + + sess.execute(self.tables.parent.delete()) + + p1.data = 3 + assert_raises_message( + orm_exc.StaleDataError, + "UPDATE statement on table 'parent' expected to " + "update 1 row\(s\); 0 were matched.", + sess.flush + ) + + def test_update_multi_missing_broken_multi_rowcount(self): + @util.memoized_property + def rowcount(self): + if len(self.context.compiled_parameters) > 1: + return -1 + else: + return self.context.rowcount + + with patch.object( + config.db.dialect, "supports_sane_multi_rowcount", False): + with patch( + "sqlalchemy.engine.result.ResultProxy.rowcount", + rowcount): + Parent, Child = self._fixture() + sess = Session() + p1 = Parent(id=1, data=2) + p2 = Parent(id=2, data=3) + sess.add_all([p1, p2]) + sess.flush() + + sess.execute(self.tables.parent.delete().where(Parent.id == 1)) + + p1.data = 3 + p2.data = 4 + sess.flush() # no exception + + # update occurred for remaining row + eq_( + sess.query(Parent.id, Parent.data).all(), + [(2, 4)] + ) + @testing.requires.sane_multi_rowcount def test_delete_multi_missing_warning(self): Parent, Child = self._fixture() @@ -1544,6 +1605,7 @@ class BatchInsertsTest(fixtures.MappedTest, testing.AssertsExecutionResults): T(id=10, data='t10', def_='def3'), T(id=11, data='t11'), ]) + self.assert_sql_execution( testing.db, sess.flush, -- cgit v1.2.1 From f5d4f2685f30817af493c32d2cf0ac77715bdb46 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 18 Jan 2015 20:57:26 -0500 Subject: - rework assertsql system, fixes #3293 --- lib/sqlalchemy/testing/assertions.py | 11 +- lib/sqlalchemy/testing/assertsql.py | 520 ++++++++++++++++------------------ lib/sqlalchemy/testing/fixtures.py | 5 +- test/dialect/mssql/test_query.py | 76 ++++- test/dialect/postgresql/test_query.py | 288 +++++++++++-------- test/dialect/postgresql/test_types.py | 8 +- test/orm/test_cycles.py | 10 +- test/orm/test_query.py | 9 +- test/sql/test_constraints.py | 12 +- 9 files changed, 484 insertions(+), 455 deletions(-) diff --git a/lib/sqlalchemy/testing/assertions.py b/lib/sqlalchemy/testing/assertions.py index 46fcd64b1..635f6c539 100644 --- a/lib/sqlalchemy/testing/assertions.py +++ b/lib/sqlalchemy/testing/assertions.py @@ -419,21 +419,16 @@ class AssertsExecutionResults(object): callable_() asserter.assert_(*rules) - def assert_sql(self, db, callable_, list_, with_sequences=None): - if (with_sequences is not None and - config.db.dialect.supports_sequences): - rules = with_sequences - else: - rules = list_ + def assert_sql(self, db, callable_, rules): newrules = [] for rule in rules: if isinstance(rule, dict): newrule = assertsql.AllOf(*[ - assertsql.ExactSQL(k, v) for k, v in rule.items() + assertsql.CompiledSQL(k, v) for k, v in rule.items() ]) else: - newrule = assertsql.ExactSQL(*rule) + newrule = assertsql.CompiledSQL(*rule) newrules.append(newrule) self.assert_sql_execution(db, callable_, *newrules) diff --git a/lib/sqlalchemy/testing/assertsql.py b/lib/sqlalchemy/testing/assertsql.py index 2ac0605a2..5c746e8f1 100644 --- a/lib/sqlalchemy/testing/assertsql.py +++ b/lib/sqlalchemy/testing/assertsql.py @@ -11,84 +11,138 @@ import re import collections import contextlib from .. import event +from sqlalchemy.schema import _DDLCompiles +from sqlalchemy.engine.util import _distill_params class AssertRule(object): - def process_execute(self, clauseelement, *multiparams, **params): - pass + is_consumed = False + errormessage = None + consume_statement = True - def process_cursor_execute(self, statement, parameters, context, - executemany): + def process_statement(self, execute_observed): pass - def is_consumed(self): - """Return True if this rule has been consumed, False if not. - - Should raise an AssertionError if this rule's condition has - definitely failed. - - """ - - raise NotImplementedError() + def no_more_statements(self): + assert False, 'All statements are complete, but pending '\ + 'assertion rules remain' - def rule_passed(self): - """Return True if the last test of this rule passed, False if - failed, None if no test was applied.""" - raise NotImplementedError() - - def consume_final(self): - """Return True if this rule has been consumed. - - Should raise an AssertionError if this rule's condition has not - been consumed or has failed. +class SQLMatchRule(AssertRule): + pass - """ - if self._result is None: - assert False, 'Rule has not been consumed' - return self.is_consumed() +class CursorSQL(SQLMatchRule): + consume_statement = False + def __init__(self, statement, params=None): + self.statement = statement + self.params = params -class SQLMatchRule(AssertRule): - def __init__(self): - self._result = None - self._errmsg = "" + def process_statement(self, execute_observed): + stmt = execute_observed.statements[0] + if self.statement != stmt.statement or ( + self.params is not None and self.params != stmt.parameters): + self.errormessage = \ + "Testing for exact SQL %s parameters %s received %s %s" % ( + self.statement, self.params, + stmt.statement, stmt.parameters + ) + else: + execute_observed.statements.pop(0) + self.is_consumed = True + if not execute_observed.statements: + self.consume_statement = True - def rule_passed(self): - return self._result - def is_consumed(self): - if self._result is None: - return False +class CompiledSQL(SQLMatchRule): - assert self._result, self._errmsg + def __init__(self, statement, params=None): + self.statement = statement + self.params = params - return True + def _compare_sql(self, execute_observed, received_statement): + stmt = re.sub(r'[\n\t]', '', self.statement) + return received_statement == stmt + def _compile_dialect(self, execute_observed): + return DefaultDialect() -class ExactSQL(SQLMatchRule): + def _received_statement(self, execute_observed): + """reconstruct the statement and params in terms + of a target dialect, which for CompiledSQL is just DefaultDialect.""" - def __init__(self, sql, params=None): - SQLMatchRule.__init__(self) - self.sql = sql - self.params = params + context = execute_observed.context + compare_dialect = self._compile_dialect(execute_observed) + if isinstance(context.compiled.statement, _DDLCompiles): + compiled = \ + context.compiled.statement.compile(dialect=compare_dialect) + else: + compiled = ( + context.compiled.statement.compile( + dialect=compare_dialect, + column_keys=context.compiled.column_keys, + inline=context.compiled.inline) + ) + _received_statement = re.sub(r'[\n\t]', '', str(compiled)) + parameters = execute_observed.parameters - def process_cursor_execute(self, statement, parameters, context, - executemany): - if not context: - return - _received_statement = \ - _process_engine_statement(context.unicode_statement, - context) - _received_parameters = context.compiled_parameters + if not parameters: + _received_parameters = [compiled.construct_params()] + else: + _received_parameters = [ + compiled.construct_params(m) for m in parameters] + + return _received_statement, _received_parameters + + def process_statement(self, execute_observed): + context = execute_observed.context + + _received_statement, _received_parameters = \ + self._received_statement(execute_observed) + params = self._all_params(context) + + equivalent = self._compare_sql(execute_observed, _received_statement) + + if equivalent: + if params is not None: + all_params = list(params) + all_received = list(_received_parameters) + while all_params and all_received: + param = dict(all_params.pop(0)) + + for idx, received in enumerate(list(all_received)): + # do a positive compare only + for param_key in param: + # a key in param did not match current + # 'received' + if param_key not in received or \ + received[param_key] != param[param_key]: + break + else: + # all keys in param matched 'received'; + # onto next param + del all_received[idx] + break + else: + # param did not match any entry + # in all_received + equivalent = False + break + if all_params or all_received: + equivalent = False - # TODO: remove this step once all unit tests are migrated, as - # ExactSQL should really be *exact* SQL + if equivalent: + self.is_consumed = True + self.errormessage = None + else: + self.errormessage = self._failure_message(params) % { + 'received_statement': _received_statement, + 'received_parameters': _received_parameters + } - sql = _process_assertion_statement(self.sql, context) - equivalent = _received_statement == sql + def _all_params(self, context): if self.params: if util.callable(self.params): params = self.params(context) @@ -96,127 +150,77 @@ class ExactSQL(SQLMatchRule): params = self.params if not isinstance(params, list): params = [params] - equivalent = equivalent and params \ - == context.compiled_parameters + return params else: - params = {} - self._result = equivalent - if not self._result: - self._errmsg = ( - 'Testing for exact statement %r exact params %r, ' - 'received %r with params %r' % - (sql, params, _received_statement, _received_parameters)) - + return None + + def _failure_message(self, expected_params): + return ( + 'Testing for compiled statement %r partial params %r, ' + 'received %%(received_statement)r with params ' + '%%(received_parameters)r' % ( + self.statement, expected_params + ) + ) -class RegexSQL(SQLMatchRule): +class RegexSQL(CompiledSQL): def __init__(self, regex, params=None): SQLMatchRule.__init__(self) self.regex = re.compile(regex) self.orig_regex = regex self.params = params - def process_cursor_execute(self, statement, parameters, context, - executemany): - if not context: - return - _received_statement = \ - _process_engine_statement(context.unicode_statement, - context) - _received_parameters = context.compiled_parameters - equivalent = bool(self.regex.match(_received_statement)) - if self.params: - if util.callable(self.params): - params = self.params(context) - else: - params = self.params - if not isinstance(params, list): - params = [params] - - # do a positive compare only - - for param, received in zip(params, _received_parameters): - for k, v in param.items(): - if k not in received or received[k] != v: - equivalent = False - break - else: - params = {} - self._result = equivalent - if not self._result: - self._errmsg = \ - 'Testing for regex %r partial params %r, received %r '\ - 'with params %r' % (self.orig_regex, params, - _received_statement, - _received_parameters) - - -class CompiledSQL(SQLMatchRule): + def _failure_message(self, expected_params): + return ( + 'Testing for compiled statement ~%r partial params %r, ' + 'received %%(received_statement)r with params ' + '%%(received_parameters)r' % ( + self.orig_regex, expected_params + ) + ) - def __init__(self, statement, params=None): - SQLMatchRule.__init__(self) - self.statement = statement - self.params = params + def _compare_sql(self, execute_observed, received_statement): + return bool(self.regex.match(received_statement)) - def process_cursor_execute(self, statement, parameters, context, - executemany): - if not context: - return - from sqlalchemy.schema import _DDLCompiles - _received_parameters = list(context.compiled_parameters) - # recompile from the context, using the default dialect +class DialectSQL(CompiledSQL): + def _compile_dialect(self, execute_observed): + return execute_observed.context.dialect - if isinstance(context.compiled.statement, _DDLCompiles): - compiled = \ - context.compiled.statement.compile(dialect=DefaultDialect()) + def _received_statement(self, execute_observed): + received_stmt, received_params = super(DialectSQL, self).\ + _received_statement(execute_observed) + for real_stmt in execute_observed.statements: + if real_stmt.statement == received_stmt: + break else: - compiled = ( - context.compiled.statement.compile( - dialect=DefaultDialect(), - column_keys=context.compiled.column_keys) - ) - _received_statement = re.sub(r'[\n\t]', '', str(compiled)) - equivalent = self.statement == _received_statement - if self.params: - if util.callable(self.params): - params = self.params(context) - else: - params = self.params - if not isinstance(params, list): - params = [params] - else: - params = list(params) - all_params = list(params) - all_received = list(_received_parameters) - while params: - param = dict(params.pop(0)) - for k, v in context.compiled.params.items(): - param.setdefault(k, v) - if param not in _received_parameters: - equivalent = False - break - else: - _received_parameters.remove(param) - if _received_parameters: - equivalent = False + raise AssertionError( + "Can't locate compiled statement %r in list of " + "statements actually invoked" % received_stmt) + return received_stmt, execute_observed.context.compiled_parameters + + def _compare_sql(self, execute_observed, received_statement): + stmt = re.sub(r'[\n\t]', '', self.statement) + + # convert our comparison statement to have the + # paramstyle of the received + paramstyle = execute_observed.context.dialect.paramstyle + if paramstyle == 'pyformat': + stmt = re.sub( + r':([\w_]+)', r"%(\1)s", stmt) else: - params = {} - all_params = {} - all_received = [] - self._result = equivalent - if not self._result: - print('Testing for compiled statement %r partial params ' - '%r, received %r with params %r' % - (self.statement, all_params, - _received_statement, all_received)) - self._errmsg = ( - 'Testing for compiled statement %r partial params %r, ' - 'received %r with params %r' % - (self.statement, all_params, - _received_statement, all_received)) - - # print self._errmsg + # positional params + repl = None + if paramstyle == 'qmark': + repl = "?" + elif paramstyle == 'format': + repl = r"%s" + elif paramstyle == 'numeric': + repl = None + stmt = re.sub(r':([\w_]+)', repl, stmt) + + return received_statement == stmt class CountStatements(AssertRule): @@ -225,21 +229,13 @@ class CountStatements(AssertRule): self.count = count self._statement_count = 0 - def process_execute(self, clauseelement, *multiparams, **params): + def process_statement(self, execute_observed): self._statement_count += 1 - def process_cursor_execute(self, statement, parameters, context, - executemany): - pass - - def is_consumed(self): - return False - - def consume_final(self): - assert self.count == self._statement_count, \ - 'desired statement count %d does not match %d' \ - % (self.count, self._statement_count) - return True + def no_more_statements(self): + if self.count != self._statement_count: + assert False, 'desired statement count %d does not match %d' \ + % (self.count, self._statement_count) class AllOf(AssertRule): @@ -247,98 +243,41 @@ class AllOf(AssertRule): def __init__(self, *rules): self.rules = set(rules) - def process_execute(self, clauseelement, *multiparams, **params): - for rule in self.rules: - rule.process_execute(clauseelement, *multiparams, **params) - - def process_cursor_execute(self, statement, parameters, context, - executemany): - for rule in self.rules: - rule.process_cursor_execute(statement, parameters, context, - executemany) - - def is_consumed(self): - if not self.rules: - return True + def process_statement(self, execute_observed): for rule in list(self.rules): - if rule.rule_passed(): # a rule passed, move on - self.rules.remove(rule) - return len(self.rules) == 0 - return False - - def rule_passed(self): - return self.is_consumed() - - def consume_final(self): - return len(self.rules) == 0 + rule.errormessage = None + rule.process_statement(execute_observed) + if rule.is_consumed: + self.rules.discard(rule) + if not self.rules: + self.is_consumed = True + break + elif not rule.errormessage: + # rule is not done yet + self.errormessage = None + break + else: + self.errormessage = list(self.rules)[0].errormessage class Or(AllOf): - def __init__(self, *rules): - self.rules = set(rules) - self._consume_final = False - - def is_consumed(self): - if not self.rules: - return True - for rule in list(self.rules): - if rule.rule_passed(): # a rule passed - self._consume_final = True - return True - return False - - def consume_final(self): - assert self._consume_final, "Unsatisified rules remain" - - -def _process_engine_statement(query, context): - if util.jython: - - # oracle+zxjdbc passes a PyStatement when returning into - - query = str(query) - if context.engine.name == 'mssql' \ - and query.endswith('; select scope_identity()'): - query = query[:-25] - query = re.sub(r'\n', '', query) - return query + def process_statement(self, execute_observed): + for rule in self.rules: + rule.process_statement(execute_observed) + if rule.is_consumed: + self.is_consumed = True + break + else: + self.errormessage = list(self.rules)[0].errormessage -def _process_assertion_statement(query, context): - paramstyle = context.dialect.paramstyle - if paramstyle == 'named': - pass - elif paramstyle == 'pyformat': - query = re.sub(r':([\w_]+)', r"%(\1)s", query) - else: - # positional params - repl = None - if paramstyle == 'qmark': - repl = "?" - elif paramstyle == 'format': - repl = r"%s" - elif paramstyle == 'numeric': - repl = None - query = re.sub(r':([\w_]+)', repl, query) - return query - - -class SQLExecuteObserved( - collections.namedtuple( - "SQLExecuteObserved", ["clauseelement", "multiparams", "params"]) -): - def process(self, rules): - if rules is not None: - if not rules: - assert False, \ - 'All rules have been exhausted, but further '\ - 'statements remain' - rule = rules[0] - rule.process_execute( - self.clauseelement, *self.multiparams, **self.params) - if rule.is_consumed(): - rules.pop(0) +class SQLExecuteObserved(object): + def __init__(self, context, clauseelement, multiparams, params): + self.context = context + self.clauseelement = clauseelement + self.parameters = _distill_params(multiparams, params) + self.statements = [] class SQLCursorExecuteObserved( @@ -346,12 +285,7 @@ class SQLCursorExecuteObserved( "SQLCursorExecuteObserved", ["statement", "parameters", "context", "executemany"]) ): - def process(self, rules): - if rules: - rule = rules[0] - rule.process_cursor_execute( - self.statement, self.parameters, - self.context, self.executemany) + pass class SQLAsserter(object): @@ -359,43 +293,63 @@ class SQLAsserter(object): self.accumulated = [] def _close(self): - # safety feature in case event.remove - # goes haywire self._final = self.accumulated del self.accumulated def assert_(self, *rules): rules = list(rules) - for observed in self._final: - observed.process(rules) + observed = list(self._final) + + while observed and rules: + rule = rules[0] + rule.process_statement(observed[0]) + if rule.is_consumed: + rules.pop(0) + elif rule.errormessage: + assert False, rule.errormessage - for rule in rules: - if not rule.consume_final(): - assert False, \ - 'All statements are complete, but pending '\ - 'assertion rules remain' + if rule.consume_statement: + observed.pop(0) + + if not observed and rules: + rules[0].no_more_statements() + elif not rules and observed: + assert False, "Additional SQL statements remain" @contextlib.contextmanager def assert_engine(engine): asserter = SQLAsserter() - @event.listens_for(engine, "after_execute") - def execute(conn, clauseelement, multiparams, params, result): - asserter.accumulated.append( - SQLExecuteObserved( - clauseelement, multiparams, params)) + orig = [] + + @event.listens_for(engine, "before_execute") + def connection_execute(conn, clauseelement, multiparams, params): + # grab the original statement + params before any cursor + # execution + orig[:] = clauseelement, multiparams, params @event.listens_for(engine, "after_cursor_execute") def cursor_execute(conn, cursor, statement, parameters, context, executemany): - asserter.accumulated.append( + if not context: + return + # then grab real cursor statements and associate them all + # around a single context + if asserter.accumulated and \ + asserter.accumulated[-1].context is context: + obs = asserter.accumulated[-1] + else: + obs = SQLExecuteObserved(context, orig[0], orig[1], orig[2]) + asserter.accumulated.append(obs) + obs.statements.append( SQLCursorExecuteObserved( - statement, parameters, context, executemany)) + statement, parameters, context, executemany) + ) try: yield asserter finally: - asserter._close() event.remove(engine, "after_cursor_execute", cursor_execute) - event.remove(engine, "after_execute", execute) + event.remove(engine, "before_execute", connection_execute) + asserter._close() diff --git a/lib/sqlalchemy/testing/fixtures.py b/lib/sqlalchemy/testing/fixtures.py index d86049da7..48d4d9c9b 100644 --- a/lib/sqlalchemy/testing/fixtures.py +++ b/lib/sqlalchemy/testing/fixtures.py @@ -192,9 +192,8 @@ class TablesTest(TestBase): def sql_count_(self, count, fn): self.assert_sql_count(self.bind, fn, count) - def sql_eq_(self, callable_, statements, with_sequences=None): - self.assert_sql(self.bind, - callable_, statements, with_sequences) + def sql_eq_(self, callable_, statements): + self.assert_sql(self.bind, callable_, statements) @classmethod def _load_fixtures(cls): diff --git a/test/dialect/mssql/test_query.py b/test/dialect/mssql/test_query.py index 715eebb84..e0affe831 100644 --- a/test/dialect/mssql/test_query.py +++ b/test/dialect/mssql/test_query.py @@ -7,6 +7,7 @@ from sqlalchemy.testing import fixtures, AssertsCompiledSQL from sqlalchemy import testing from sqlalchemy.util import ue from sqlalchemy import util +from sqlalchemy.testing.assertsql import CursorSQL @@ -163,7 +164,6 @@ class QueryUnicodeTest(fixtures.TestBase): finally: meta.drop_all() -from sqlalchemy.testing.assertsql import ExactSQL class QueryTest(testing.AssertsExecutionResults, fixtures.TestBase): __only_on__ = 'mssql' @@ -232,27 +232,73 @@ class QueryTest(testing.AssertsExecutionResults, fixtures.TestBase): con.execute("""drop trigger paj""") meta.drop_all() - @testing.fails_on_everything_except('mssql+pyodbc', 'pyodbc-specific feature') @testing.provide_metadata def test_disable_scope_identity(self): engine = engines.testing_engine(options={"use_scope_identity": False}) metadata = self.metadata - metadata.bind = engine - t1 = Table('t1', metadata, - Column('id', Integer, primary_key=True), - implicit_returning=False + t1 = Table( + 't1', metadata, + Column('id', Integer, primary_key=True), + Column('data', String(50)), + implicit_returning=False ) - metadata.create_all() + metadata.create_all(engine) + + with self.sql_execution_asserter(engine) as asserter: + engine.execute(t1.insert(), {"data": "somedata"}) + + asserter.assert_( + CursorSQL( + "INSERT INTO t1 (data) VALUES (?)", + ("somedata", ) + ), + CursorSQL("SELECT @@identity AS lastrowid"), + ) + + @testing.provide_metadata + def test_enable_scope_identity(self): + engine = engines.testing_engine(options={"use_scope_identity": True}) + metadata = self.metadata + t1 = Table( + 't1', metadata, + Column('id', Integer, primary_key=True), + implicit_returning=False + ) + metadata.create_all(engine) + + with self.sql_execution_asserter(engine) as asserter: + engine.execute(t1.insert()) + + # even with pyodbc, we don't embed the scope identity on a + # DEFAULT VALUES insert + asserter.assert_( + CursorSQL("INSERT INTO t1 DEFAULT VALUES"), + CursorSQL("SELECT scope_identity() AS lastrowid"), + ) + + @testing.only_on('mssql+pyodbc') + @testing.provide_metadata + def test_embedded_scope_identity(self): + engine = engines.testing_engine(options={"use_scope_identity": True}) + metadata = self.metadata + t1 = Table( + 't1', metadata, + Column('id', Integer, primary_key=True), + Column('data', String(50)), + implicit_returning=False + ) + metadata.create_all(engine) + + with self.sql_execution_asserter(engine) as asserter: + engine.execute(t1.insert(), {'data': 'somedata'}) - self.assert_sql_execution( - testing.db, - lambda: engine.execute(t1.insert()), - ExactSQL("INSERT INTO t1 DEFAULT VALUES"), - # we don't have an event for - # "SELECT @@IDENTITY" part here. - # this will be in 0.8 with #2459 + # pyodbc-specific system + asserter.assert_( + CursorSQL( + "INSERT INTO t1 (data) VALUES (?); select scope_identity()", + ("somedata", ) + ), ) - assert not engine.dialect.use_scope_identity def test_insertid_schema(self): meta = MetaData(testing.db) diff --git a/test/dialect/postgresql/test_query.py b/test/dialect/postgresql/test_query.py index 6841f397a..26ff5e93b 100644 --- a/test/dialect/postgresql/test_query.py +++ b/test/dialect/postgresql/test_query.py @@ -6,6 +6,7 @@ from sqlalchemy import Table, Column, MetaData, Integer, String, bindparam, \ Sequence, ForeignKey, text, select, func, extract, literal_column, \ tuple_, DateTime, Time, literal, and_, Date, or_ from sqlalchemy.testing import engines, fixtures +from sqlalchemy.testing.assertsql import DialectSQL, CursorSQL from sqlalchemy import testing from sqlalchemy import exc from sqlalchemy.dialects import postgresql @@ -170,7 +171,7 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): engines.testing_engine(options={'implicit_returning': False}) metadata.bind = self.engine - def go(): + with self.sql_execution_asserter(self.engine) as asserter: # execute with explicit id @@ -199,32 +200,41 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): table.insert(inline=True).execute({'data': 'd8'}) - # note that the test framework doesn't capture the "preexecute" - # of a seqeuence or default. we just see it in the bind params. + asserter.assert_( + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + {'id': 30, 'data': 'd1'}), + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + {'id': 1, 'data': 'd2'}), + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), + DialectSQL( + 'INSERT INTO testtable (data) VALUES (:data)', + [{'data': 'd5'}, {'data': 'd6'}]), + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + [{'id': 33, 'data': 'd7'}]), + DialectSQL( + 'INSERT INTO testtable (data) VALUES (:data)', + [{'data': 'd8'}]), + ) + + eq_( + table.select().execute().fetchall(), + [ + (30, 'd1'), + (1, 'd2'), + (31, 'd3'), + (32, 'd4'), + (2, 'd5'), + (3, 'd6'), + (33, 'd7'), + (4, 'd8'), + ] + ) - self.assert_sql(self.engine, go, [], with_sequences=[ - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - {'id': 30, 'data': 'd1'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - {'id': 1, 'data': 'd2'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), - ('INSERT INTO testtable (data) VALUES (:data)', - [{'data': 'd5'}, {'data': 'd6'}]), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - [{'id': 33, 'data': 'd7'}]), - ('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd8'}]), - ]) - assert table.select().execute().fetchall() == [ - (30, 'd1'), - (1, 'd2'), - (31, 'd3'), - (32, 'd4'), - (2, 'd5'), - (3, 'd6'), - (33, 'd7'), - (4, 'd8'), - ] table.delete().execute() # test the same series of events using a reflected version of @@ -233,7 +243,7 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): m2 = MetaData(self.engine) table = Table(table.name, m2, autoload=True) - def go(): + with self.sql_execution_asserter(self.engine) as asserter: table.insert().execute({'id': 30, 'data': 'd1'}) r = table.insert().execute({'data': 'd2'}) assert r.inserted_primary_key == [5] @@ -243,29 +253,39 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): table.insert(inline=True).execute({'id': 33, 'data': 'd7'}) table.insert(inline=True).execute({'data': 'd8'}) - self.assert_sql(self.engine, go, [], with_sequences=[ - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - {'id': 30, 'data': 'd1'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - {'id': 5, 'data': 'd2'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), - ('INSERT INTO testtable (data) VALUES (:data)', - [{'data': 'd5'}, {'data': 'd6'}]), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', - [{'id': 33, 'data': 'd7'}]), - ('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd8'}]), - ]) - assert table.select().execute().fetchall() == [ - (30, 'd1'), - (5, 'd2'), - (31, 'd3'), - (32, 'd4'), - (6, 'd5'), - (7, 'd6'), - (33, 'd7'), - (8, 'd8'), - ] + asserter.assert_( + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + {'id': 30, 'data': 'd1'}), + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + {'id': 5, 'data': 'd2'}), + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), + DialectSQL( + 'INSERT INTO testtable (data) VALUES (:data)', + [{'data': 'd5'}, {'data': 'd6'}]), + DialectSQL( + 'INSERT INTO testtable (id, data) VALUES (:id, :data)', + [{'id': 33, 'data': 'd7'}]), + DialectSQL( + 'INSERT INTO testtable (data) VALUES (:data)', + [{'data': 'd8'}]), + ) + eq_( + table.select().execute().fetchall(), + [ + (30, 'd1'), + (5, 'd2'), + (31, 'd3'), + (32, 'd4'), + (6, 'd5'), + (7, 'd6'), + (33, 'd7'), + (8, 'd8'), + ] + ) table.delete().execute() def _assert_data_autoincrement_returning(self, table): @@ -273,7 +293,7 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): engines.testing_engine(options={'implicit_returning': True}) metadata.bind = self.engine - def go(): + with self.sql_execution_asserter(self.engine) as asserter: # execute with explicit id @@ -302,29 +322,34 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): table.insert(inline=True).execute({'data': 'd8'}) - self.assert_sql(self.engine, go, [], with_sequences=[ - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + asserter.assert_( + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', {'id': 30, 'data': 'd1'}), - ('INSERT INTO testtable (data) VALUES (:data) RETURNING ' + DialectSQL('INSERT INTO testtable (data) VALUES (:data) RETURNING ' 'testtable.id', {'data': 'd2'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), - ('INSERT INTO testtable (data) VALUES (:data)', + DialectSQL('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd5'}, {'data': 'd6'}]), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 33, 'data': 'd7'}]), - ('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd8'}]), - ]) - assert table.select().execute().fetchall() == [ - (30, 'd1'), - (1, 'd2'), - (31, 'd3'), - (32, 'd4'), - (2, 'd5'), - (3, 'd6'), - (33, 'd7'), - (4, 'd8'), - ] + DialectSQL('INSERT INTO testtable (data) VALUES (:data)', + [{'data': 'd8'}]), + ) + + eq_( + table.select().execute().fetchall(), + [ + (30, 'd1'), + (1, 'd2'), + (31, 'd3'), + (32, 'd4'), + (2, 'd5'), + (3, 'd6'), + (33, 'd7'), + (4, 'd8'), + ] + ) table.delete().execute() # test the same series of events using a reflected version of @@ -333,7 +358,7 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): m2 = MetaData(self.engine) table = Table(table.name, m2, autoload=True) - def go(): + with self.sql_execution_asserter(self.engine) as asserter: table.insert().execute({'id': 30, 'data': 'd1'}) r = table.insert().execute({'data': 'd2'}) assert r.inserted_primary_key == [5] @@ -343,29 +368,32 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): table.insert(inline=True).execute({'id': 33, 'data': 'd7'}) table.insert(inline=True).execute({'data': 'd8'}) - self.assert_sql(self.engine, go, [], with_sequences=[ - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + asserter.assert_( + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', {'id': 30, 'data': 'd1'}), - ('INSERT INTO testtable (data) VALUES (:data) RETURNING ' + DialectSQL('INSERT INTO testtable (data) VALUES (:data) RETURNING ' 'testtable.id', {'data': 'd2'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), - ('INSERT INTO testtable (data) VALUES (:data)', + DialectSQL('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd5'}, {'data': 'd6'}]), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 33, 'data': 'd7'}]), - ('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd8'}]), - ]) - assert table.select().execute().fetchall() == [ - (30, 'd1'), - (5, 'd2'), - (31, 'd3'), - (32, 'd4'), - (6, 'd5'), - (7, 'd6'), - (33, 'd7'), - (8, 'd8'), - ] + DialectSQL('INSERT INTO testtable (data) VALUES (:data)', [{'data': 'd8'}]), + ) + eq_( + table.select().execute().fetchall(), + [ + (30, 'd1'), + (5, 'd2'), + (31, 'd3'), + (32, 'd4'), + (6, 'd5'), + (7, 'd6'), + (33, 'd7'), + (8, 'd8'), + ] + ) table.delete().execute() def _assert_data_with_sequence(self, table, seqname): @@ -373,7 +401,7 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): engines.testing_engine(options={'implicit_returning': False}) metadata.bind = self.engine - def go(): + with self.sql_execution_asserter(self.engine) as asserter: table.insert().execute({'id': 30, 'data': 'd1'}) table.insert().execute({'data': 'd2'}) table.insert().execute({'id': 31, 'data': 'd3'}, {'id': 32, @@ -382,30 +410,34 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): table.insert(inline=True).execute({'id': 33, 'data': 'd7'}) table.insert(inline=True).execute({'data': 'd8'}) - self.assert_sql(self.engine, go, [], with_sequences=[ - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + asserter.assert_( + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', {'id': 30, 'data': 'd1'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + CursorSQL("select nextval('my_seq')"), + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', {'id': 1, 'data': 'd2'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), - ("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " + DialectSQL("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " ":data)" % seqname, [{'data': 'd5'}, {'data': 'd6'}]), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 33, 'data': 'd7'}]), - ("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " + DialectSQL("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " ":data)" % seqname, [{'data': 'd8'}]), - ]) - assert table.select().execute().fetchall() == [ - (30, 'd1'), - (1, 'd2'), - (31, 'd3'), - (32, 'd4'), - (2, 'd5'), - (3, 'd6'), - (33, 'd7'), - (4, 'd8'), - ] + ) + eq_( + table.select().execute().fetchall(), + [ + (30, 'd1'), + (1, 'd2'), + (31, 'd3'), + (32, 'd4'), + (2, 'd5'), + (3, 'd6'), + (33, 'd7'), + (4, 'd8'), + ] + ) # cant test reflection here since the Sequence must be # explicitly specified @@ -415,7 +447,7 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): engines.testing_engine(options={'implicit_returning': True}) metadata.bind = self.engine - def go(): + with self.sql_execution_asserter(self.engine) as asserter: table.insert().execute({'id': 30, 'data': 'd1'}) table.insert().execute({'data': 'd2'}) table.insert().execute({'id': 31, 'data': 'd3'}, {'id': 32, @@ -424,31 +456,35 @@ class InsertTest(fixtures.TestBase, AssertsExecutionResults): table.insert(inline=True).execute({'id': 33, 'data': 'd7'}) table.insert(inline=True).execute({'data': 'd8'}) - self.assert_sql(self.engine, go, [], with_sequences=[ - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + asserter.assert_( + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', {'id': 30, 'data': 'd1'}), - ("INSERT INTO testtable (id, data) VALUES " + DialectSQL("INSERT INTO testtable (id, data) VALUES " "(nextval('my_seq'), :data) RETURNING testtable.id", {'data': 'd2'}), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 31, 'data': 'd3'}, {'id': 32, 'data': 'd4'}]), - ("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " + DialectSQL("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " ":data)" % seqname, [{'data': 'd5'}, {'data': 'd6'}]), - ('INSERT INTO testtable (id, data) VALUES (:id, :data)', + DialectSQL('INSERT INTO testtable (id, data) VALUES (:id, :data)', [{'id': 33, 'data': 'd7'}]), - ("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " + DialectSQL("INSERT INTO testtable (id, data) VALUES (nextval('%s'), " ":data)" % seqname, [{'data': 'd8'}]), - ]) - assert table.select().execute().fetchall() == [ - (30, 'd1'), - (1, 'd2'), - (31, 'd3'), - (32, 'd4'), - (2, 'd5'), - (3, 'd6'), - (33, 'd7'), - (4, 'd8'), - ] + ) + + eq_( + table.select().execute().fetchall(), + [ + (30, 'd1'), + (1, 'd2'), + (31, 'd3'), + (32, 'd4'), + (2, 'd5'), + (3, 'd6'), + (33, 'd7'), + (4, 'd8'), + ] + ) # cant test reflection here since the Sequence must be # explicitly specified diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index 5c5da59b1..c62ca79a8 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -189,7 +189,7 @@ class EnumTest(fixtures.TestBase, AssertsExecutionResults): try: self.assert_sql( - testing.db, go, [], with_sequences=[ + testing.db, go, [ ("CREATE TABLE foo (\tbar " "VARCHAR(5), \tCONSTRAINT myenum CHECK " "(bar IN ('one', 'two', 'three')))", {})]) @@ -259,9 +259,9 @@ class EnumTest(fixtures.TestBase, AssertsExecutionResults): try: self.assert_sql( - engine, go, [], with_sequences=[ - ("CREATE TABLE foo (\tbar " - "VARCHAR(5), \tCONSTRAINT myenum CHECK " + engine, go, [ + ("CREATE TABLE foo (bar " + "VARCHAR(5), CONSTRAINT myenum CHECK " "(bar IN ('one', 'two', 'three')))", {})]) finally: metadata.drop_all(engine) diff --git a/test/orm/test_cycles.py b/test/orm/test_cycles.py index fc7059dcb..c95b8d152 100644 --- a/test/orm/test_cycles.py +++ b/test/orm/test_cycles.py @@ -11,7 +11,7 @@ from sqlalchemy.testing.schema import Table, Column from sqlalchemy.orm import mapper, relationship, backref, \ create_session, sessionmaker from sqlalchemy.testing import eq_ -from sqlalchemy.testing.assertsql import RegexSQL, ExactSQL, CompiledSQL, AllOf +from sqlalchemy.testing.assertsql import RegexSQL, CompiledSQL, AllOf from sqlalchemy.testing import fixtures @@ -656,7 +656,7 @@ class OneToManyManyToOneTest(fixtures.MappedTest): RegexSQL("^INSERT INTO ball", lambda c: {'person_id':p.id, 'data':'some data'}), RegexSQL("^INSERT INTO ball", lambda c: {'person_id':p.id, 'data':'some data'}), RegexSQL("^INSERT INTO ball", lambda c: {'person_id':p.id, 'data':'some data'}), - ExactSQL("UPDATE person SET favorite_ball_id=:favorite_ball_id " + CompiledSQL("UPDATE person SET favorite_ball_id=:favorite_ball_id " "WHERE person.id = :person_id", lambda ctx:{'favorite_ball_id':p.favorite.id, 'person_id':p.id} ), @@ -667,11 +667,11 @@ class OneToManyManyToOneTest(fixtures.MappedTest): self.assert_sql_execution( testing.db, sess.flush, - ExactSQL("UPDATE person SET favorite_ball_id=:favorite_ball_id " + CompiledSQL("UPDATE person SET favorite_ball_id=:favorite_ball_id " "WHERE person.id = :person_id", lambda ctx: {'person_id': p.id, 'favorite_ball_id': None}), - ExactSQL("DELETE FROM ball WHERE ball.id = :id", None), # lambda ctx:[{'id': 1L}, {'id': 4L}, {'id': 3L}, {'id': 2L}]) - ExactSQL("DELETE FROM person WHERE person.id = :id", lambda ctx:[{'id': p.id}]) + CompiledSQL("DELETE FROM ball WHERE ball.id = :id", None), # lambda ctx:[{'id': 1L}, {'id': 4L}, {'id': 3L}, {'id': 2L}]) + CompiledSQL("DELETE FROM person WHERE person.id = :id", lambda ctx:[{'id': p.id}]) ) def test_post_update_backref(self): diff --git a/test/orm/test_query.py b/test/orm/test_query.py index 354bbe5b1..4c6e16bf2 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -1484,7 +1484,6 @@ class SliceTest(QueryTest): assert create_session().query(User).filter(User.id == 27). \ first() is None - @testing.only_on('sqlite', 'testing execution but db-specific syntax') def test_limit_offset_applies(self): """Test that the expected LIMIT/OFFSET is applied for slices. @@ -1510,15 +1509,15 @@ class SliceTest(QueryTest): testing.db, lambda: q[:20], [ ( "SELECT users.id AS users_id, users.name " - "AS users_name FROM users LIMIT :param_1 OFFSET :param_2", - {'param_1': 20, 'param_2': 0})]) + "AS users_name FROM users LIMIT :param_1", + {'param_1': 20})]) self.assert_sql( testing.db, lambda: q[5:], [ ( "SELECT users.id AS users_id, users.name " - "AS users_name FROM users LIMIT :param_1 OFFSET :param_2", - {'param_1': -1, 'param_2': 5})]) + "AS users_name FROM users LIMIT -1 OFFSET :param_1", + {'param_1': 5})]) self.assert_sql(testing.db, lambda: q[2:2], []) diff --git a/test/sql/test_constraints.py b/test/sql/test_constraints.py index 604b5efeb..2603f67a3 100644 --- a/test/sql/test_constraints.py +++ b/test/sql/test_constraints.py @@ -9,7 +9,7 @@ from sqlalchemy import testing from sqlalchemy.engine import default from sqlalchemy.testing import engines from sqlalchemy.testing import eq_ -from sqlalchemy.testing.assertsql import AllOf, RegexSQL, ExactSQL, CompiledSQL +from sqlalchemy.testing.assertsql import AllOf, RegexSQL, CompiledSQL from sqlalchemy.sql import table, column @@ -417,13 +417,13 @@ class ConstraintGenTest(fixtures.TestBase, AssertsExecutionResults): lambda: events.create(testing.db), RegexSQL("^CREATE TABLE events"), AllOf( - ExactSQL('CREATE UNIQUE INDEX ix_events_name ON events ' + CompiledSQL('CREATE UNIQUE INDEX ix_events_name ON events ' '(name)'), - ExactSQL('CREATE INDEX ix_events_location ON events ' + CompiledSQL('CREATE INDEX ix_events_location ON events ' '(location)'), - ExactSQL('CREATE UNIQUE INDEX sport_announcer ON events ' + CompiledSQL('CREATE UNIQUE INDEX sport_announcer ON events ' '(sport, announcer)'), - ExactSQL('CREATE INDEX idx_winners ON events (winner)') + CompiledSQL('CREATE INDEX idx_winners ON events (winner)'), ) ) @@ -441,7 +441,7 @@ class ConstraintGenTest(fixtures.TestBase, AssertsExecutionResults): lambda: t.create(testing.db), CompiledSQL('CREATE TABLE sometable (id INTEGER NOT NULL, ' 'data VARCHAR(50), PRIMARY KEY (id))'), - ExactSQL('CREATE INDEX myindex ON sometable (data DESC)') + CompiledSQL('CREATE INDEX myindex ON sometable (data DESC)') ) -- cgit v1.2.1 From 28d0b8d1d1b135c2d6974f4d31e56b8d49c8ef09 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 18 Jan 2015 23:32:52 -0500 Subject: - fix another issue from rf49c367ef, add another test --- lib/sqlalchemy/orm/persistence.py | 2 +- test/orm/test_unitofworkv2.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index dbf1d3eb4..d76bb5598 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -677,7 +677,7 @@ def _emit_update_statements(base_mapper, uowtransaction, c.context.compiled_parameters[0], value_params) - if assert_multirow or assert_singlerow and \ + if hasvalue or assert_multirow or assert_singlerow and \ len(multiparams) == 1: if rows != len(records): raise orm_exc.StaleDataError( diff --git a/test/orm/test_unitofworkv2.py b/test/orm/test_unitofworkv2.py index 681b104cf..cef71370d 100644 --- a/test/orm/test_unitofworkv2.py +++ b/test/orm/test_unitofworkv2.py @@ -5,7 +5,7 @@ from sqlalchemy.testing.schema import Table, Column from test.orm import _fixtures from sqlalchemy import exc, util from sqlalchemy.testing import fixtures, config -from sqlalchemy import Integer, String, ForeignKey, func +from sqlalchemy import Integer, String, ForeignKey, func, literal from sqlalchemy.orm import mapper, relationship, backref, \ create_session, unitofwork, attributes,\ Session, exc as orm_exc @@ -1534,6 +1534,35 @@ class BasicStaleChecksTest(fixtures.MappedTest): [(2, 4)] ) + def test_update_value_missing_broken_multi_rowcount(self): + @util.memoized_property + def rowcount(self): + if len(self.context.compiled_parameters) > 1: + return -1 + else: + return self.context.rowcount + + with patch.object( + config.db.dialect, "supports_sane_multi_rowcount", False): + with patch( + "sqlalchemy.engine.result.ResultProxy.rowcount", + rowcount): + Parent, Child = self._fixture() + sess = Session() + p1 = Parent(id=1, data=1) + sess.add(p1) + sess.flush() + + sess.execute(self.tables.parent.delete()) + + p1.data = literal(1) + assert_raises_message( + orm_exc.StaleDataError, + "UPDATE statement on table 'parent' expected to " + "update 1 row\(s\); 0 were matched.", + sess.flush + ) + @testing.requires.sane_multi_rowcount def test_delete_multi_missing_warning(self): Parent, Child = self._fixture() -- cgit v1.2.1 From 3f84a9408064bc2064bc706a369ecb463df17789 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 19 Jan 2015 08:49:44 -0500 Subject: - another adjustment --- lib/sqlalchemy/orm/persistence.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index d76bb5598..7f81a5c99 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -677,8 +677,9 @@ def _emit_update_statements(base_mapper, uowtransaction, c.context.compiled_parameters[0], value_params) - if hasvalue or assert_multirow or assert_singlerow and \ - len(multiparams) == 1: + if hasvalue or assert_multirow or ( + assert_singlerow and + len(multiparams)) == 1: if rows != len(records): raise orm_exc.StaleDataError( "UPDATE statement on table '%s' expected to " -- cgit v1.2.1 From 6135c03230391a2230e2280f2cbb8b02d880db32 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 19 Jan 2015 11:47:28 -0500 Subject: - further fixes and even better tests for this block --- lib/sqlalchemy/orm/persistence.py | 11 ++++++++--- test/orm/test_versioning.py | 28 +++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 7f81a5c99..e553f399d 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -642,8 +642,10 @@ def _emit_update_statements(base_mapper, uowtransaction, c.context.compiled_parameters[0], value_params) rows += c.rowcount + check_rowcount = True else: if not allow_multirow: + check_rowcount = assert_singlerow for state, state_dict, params, mapper, \ connection, value_params in records: c = cached_connections[connection].\ @@ -661,6 +663,11 @@ def _emit_update_statements(base_mapper, uowtransaction, else: multiparams = [rec[2] for rec in records] + check_rowcount = assert_multirow or ( + assert_singlerow and + len(multiparams) == 1 + ) + c = cached_connections[connection].\ execute(statement, multiparams) @@ -677,9 +684,7 @@ def _emit_update_statements(base_mapper, uowtransaction, c.context.compiled_parameters[0], value_params) - if hasvalue or assert_multirow or ( - assert_singlerow and - len(multiparams)) == 1: + if check_rowcount: if rows != len(records): raise orm_exc.StaleDataError( "UPDATE statement on table '%s' expected to " diff --git a/test/orm/test_versioning.py b/test/orm/test_versioning.py index 55ce586b5..8348cb588 100644 --- a/test/orm/test_versioning.py +++ b/test/orm/test_versioning.py @@ -1,7 +1,8 @@ import datetime import sqlalchemy as sa -from sqlalchemy.testing import engines +from sqlalchemy.testing import engines, config from sqlalchemy import testing +from sqlalchemy.testing.mock import patch from sqlalchemy import ( Integer, String, Date, ForeignKey, orm, exc, select, TypeDecorator) from sqlalchemy.testing.schema import Table, Column @@ -12,6 +13,7 @@ from sqlalchemy.testing import ( eq_, assert_raises, assert_raises_message, fixtures) from sqlalchemy.testing.assertsql import CompiledSQL import uuid +from sqlalchemy import util def make_uuid(): @@ -223,6 +225,30 @@ class VersioningTest(fixtures.MappedTest): s1.refresh(f1s1, lockmode='update_nowait') assert f1s1.version_id == f1s2.version_id + def test_update_multi_missing_broken_multi_rowcount(self): + @util.memoized_property + def rowcount(self): + if len(self.context.compiled_parameters) > 1: + return -1 + else: + return self.context.rowcount + + with patch.object( + config.db.dialect, "supports_sane_multi_rowcount", False): + with patch( + "sqlalchemy.engine.result.ResultProxy.rowcount", + rowcount): + + Foo = self.classes.Foo + s1 = self._fixture() + f1s1 = Foo(value='f1 value') + s1.add(f1s1) + s1.commit() + + f1s1.value = 'f2 value' + s1.flush() + eq_(f1s1.version_id, 2) + @testing.emits_warning(r'.*does not support updated rowcount') @engines.close_open_connections def test_noversioncheck(self): -- cgit v1.2.1 From dacfe7dec2d940ced2c96ef313404ab137bb69f6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 19 Jan 2015 17:29:48 -0500 Subject: - tests --- test/orm/test_query.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/test/orm/test_query.py b/test/orm/test_query.py index 4c6e16bf2..af6d960f5 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -17,6 +17,9 @@ from sqlalchemy.testing.assertions import ( from sqlalchemy.testing import fixtures, AssertsCompiledSQL, assert_warnings from test.orm import _fixtures from sqlalchemy.orm.util import join, with_parent +import contextlib +from sqlalchemy.testing import mock, is_, is_not_ +from sqlalchemy import inspect class QueryTest(_fixtures.FixtureTest): @@ -3212,3 +3215,63 @@ class BooleanEvalTest(fixtures.TestBase, testing.AssertsCompiledSQL): "SELECT x HAVING x = 1", dialect=self._dialect(False) ) + + +class SessionBindTest(QueryTest): + + @contextlib.contextmanager + def _assert_bind_args(self, session): + get_bind = mock.Mock(side_effect=session.get_bind) + with mock.patch.object(session, "get_bind", get_bind): + yield + is_(get_bind.mock_calls[0][1][0], inspect(self.classes.User)) + is_not_(get_bind.mock_calls[0][2]['clause'], None) + + def test_single_entity_q(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).all() + + def test_sql_expr_entity_q(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User.id).all() + + def test_count(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).count() + + def test_aggregate_fn(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(func.max(User.name)).all() + + def test_bulk_update(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).filter(User.id == 15).update( + {"name": "foob"}, synchronize_session=False) + + def test_bulk_delete(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).filter(User.id == 15).delete( + synchronize_session=False) + + def test_column_property(self): + User = self.classes.User + + mapper = inspect(User) + mapper.add_property( + "score", + column_property(func.coalesce(self.tables.users.c.name, None))) + session = Session() + with self._assert_bind_args(session): + session.query(func.max(User.score)).scalar() -- cgit v1.2.1 From 611883ffb35ca6664649f6328ae896c80f499780 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 19 Jan 2015 17:55:23 -0500 Subject: - The primary :class:`.Mapper` of a :class:`.Query` is now passed to the :meth:`.Session.get_bind` method when calling upon :meth:`.Query.count`, :meth:`.Query.update`, :meth:`.Query.delete`, as well as queries against mapped columns, :obj:`.column_property` objects, and SQL functions and expressions derived from mapped columns. This allows sessions that rely upon either customized :meth:`.Session.get_bind` schemes or "bound" metadata to work in all relevant cases. fixes #3227 fixes #3242 fixes #1326 --- doc/build/changelog/changelog_10.rst | 17 ++++++++++++ doc/build/changelog/migration_10.rst | 53 +++++++++++++++++++++++++++++++++++ lib/sqlalchemy/orm/persistence.py | 12 +++++--- lib/sqlalchemy/orm/query.py | 54 +++++++++++++++++++++++------------- test/orm/test_query.py | 23 ++++++++++++--- 5 files changed, 131 insertions(+), 28 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 089c9fafb..79e43e6a3 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,23 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, orm + :tickets: 3227, 3242, 1326 + + The primary :class:`.Mapper` of a :class:`.Query` is now passed to the + :meth:`.Session.get_bind` method when calling upon + :meth:`.Query.count`, :meth:`.Query.update`, :meth:`.Query.delete`, + as well as queries against mapped columns, + :obj:`.column_property` objects, and SQL functions and expressions + derived from mapped columns. This allows sessions that rely upon + either customized :meth:`.Session.get_bind` schemes or "bound" metadata + to work in all relevant cases. + + .. seealso:: + + :ref:`bug_3227` + .. change:: :tags: enhancement, sql :tickets: 3074 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index bd878f4cb..c0369d8b8 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -381,6 +381,59 @@ of inheritance-oriented scenarios, including: :ticket:`3035` + +.. _bug_3227: + +Session.get_bind() will receive the Mapper in all relevant Query cases +----------------------------------------------------------------------- + +A series of issues were repaired where the :meth:`.Session.get_bind` +would not receive the primary :class:`.Mapper` of the :class:`.Query`, +even though this mapper was readily available (the primary mapper is the +single mapper, or alternatively the first mapper, that is associated with +a :class:`.Query` object). + +The :class:`.Mapper` object, when passed to :meth:`.Session.get_bind`, +is typically used by sessions that make use of the +:paramref:`.Session.binds` parameter to associate mappers with a +series of engines (although in this use case, things frequently +"worked" in most cases anyway as the bind would be located via the +mapped table object), or more specifically implement a user-defined +:meth:`.Session.get_bind` method that provies some pattern of +selecting engines based on mappers, such as horizontal sharding or a +so-called "routing" session that routes queries to different backends. + +These scenarios include: + +* :meth:`.Query.count`:: + + session.query(User).count() + +* :meth:`.Query.update` and :meth:`.Query.delete`, both for the UPDATE/DELETE + statement as well as for the SELECT used by the "fetch" strategy:: + + session.query(User).filter(User.id == 15).update( + {"name": "foob"}, synchronize_session='fetch') + + session.query(User).filter(User.id == 15).delete( + synchronize_session='fetch') + +* Queries against individual columns:: + + session.query(User.id, User.name).all() + +* SQL functions and other expressions against indirect mappings such as + :obj:`.column_property`:: + + class User(Base): + # ... + + score = column_property(func.coalesce(self.tables.users.c.name, None))) + + session.query(func.max(User.score)).scalar() + +:ticket:`3227` :ticket:`3242` :ticket:`1326` + .. _feature_2963: .info dictionary improvements diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index e553f399d..c3b2d7bcb 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -1030,6 +1030,7 @@ class BulkUD(object): def __init__(self, query): self.query = query.enable_eagerloads(False) + self.mapper = self.query._bind_mapper() @property def session(self): @@ -1124,6 +1125,7 @@ class BulkFetch(BulkUD): self.primary_table.primary_key) self.matched_rows = session.execute( select_stmt, + mapper=self.mapper, params=query._params).fetchall() @@ -1134,7 +1136,6 @@ class BulkUpdate(BulkUD): super(BulkUpdate, self).__init__(query) self.query._no_select_modifiers("update") self.values = values - self.mapper = self.query._mapper_zero_or_none() @classmethod def factory(cls, query, synchronize_session, values): @@ -1180,7 +1181,8 @@ class BulkUpdate(BulkUD): self.context.whereclause, values) self.result = self.query.session.execute( - update_stmt, params=self.query._params) + update_stmt, params=self.query._params, + mapper=self.mapper) self.rowcount = self.result.rowcount def _do_post(self): @@ -1207,8 +1209,10 @@ class BulkDelete(BulkUD): delete_stmt = sql.delete(self.primary_table, self.context.whereclause) - self.result = self.query.session.execute(delete_stmt, - params=self.query._params) + self.result = self.query.session.execute( + delete_stmt, + params=self.query._params, + mapper=self.mapper) self.rowcount = self.result.rowcount def _do_post(self): diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 7302574e6..cd8b0efbe 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -146,7 +146,7 @@ class Query(object): ext_info, aliased_adapter ) - ent.setup_entity(*d[entity]) + ent.setup_entity(ent, *d[entity]) def _mapper_loads_polymorphically_with(self, mapper, adapter): for m2 in mapper._with_polymorphic_mappers or [mapper]: @@ -160,7 +160,6 @@ class Query(object): for from_obj in obj: info = inspect(from_obj) - if hasattr(info, 'mapper') and \ (info.is_mapper or info.is_aliased_class): self._select_from_entity = from_obj @@ -286,8 +285,9 @@ class Query(object): return self._entities[0] def _mapper_zero(self): - return self._select_from_entity or \ - self._entity_zero().entity_zero + return self._select_from_entity \ + if self._select_from_entity is not None \ + else self._entity_zero().entity_zero @property def _mapper_entities(self): @@ -301,11 +301,14 @@ class Query(object): self._mapper_zero() ) - def _mapper_zero_or_none(self): - if self._primary_entity: - return self._primary_entity.mapper - else: - return None + def _bind_mapper(self): + ezero = self._mapper_zero() + if ezero is not None: + insp = inspect(ezero) + if hasattr(insp, 'mapper'): + return insp.mapper + + return None def _only_mapper_zero(self, rationale=None): if len(self._entities) > 1: @@ -988,6 +991,7 @@ class Query(object): statement.correlate(None) q = self._from_selectable(fromclause) q._enable_single_crit = False + q._select_from_entity = self._mapper_zero() if entities: q._set_entities(entities) return q @@ -2526,7 +2530,7 @@ class Query(object): def _execute_and_instances(self, querycontext): conn = self._connection_from_session( - mapper=self._mapper_zero_or_none(), + mapper=self._bind_mapper(), clause=querycontext.statement, close_with_result=True) @@ -3160,7 +3164,7 @@ class _MapperEntity(_QueryEntity): supports_single_entity = True - def setup_entity(self, ext_info, aliased_adapter): + def setup_entity(self, original_entity, ext_info, aliased_adapter): self.mapper = ext_info.mapper self.aliased_adapter = aliased_adapter self.selectable = ext_info.selectable @@ -3507,9 +3511,9 @@ class _BundleEntity(_QueryEntity): for ent in self._entities: ent.adapt_to_selectable(c, sel) - def setup_entity(self, ext_info, aliased_adapter): + def setup_entity(self, original_entity, ext_info, aliased_adapter): for ent in self._entities: - ent.setup_entity(ext_info, aliased_adapter) + ent.setup_entity(original_entity, ext_info, aliased_adapter) def setup_context(self, query, context): for ent in self._entities: @@ -3592,15 +3596,23 @@ class _ColumnEntity(_QueryEntity): # leaking out their entities into the main select construct self.actual_froms = actual_froms = set(column._from_objects) - self.entities = util.OrderedSet( - elem._annotations['parententity'] - for elem in visitors.iterate(column, {}) + all_elements = [ + elem for elem in visitors.iterate(column, {}) if 'parententity' in elem._annotations - and actual_froms.intersection(elem._from_objects) + ] + + self.entities = util.unique_list([ + elem._annotations['parententity'] + for elem in all_elements + ]) + self._from_entities = set( + elem._annotations['parententity'] + for elem in all_elements + if actual_froms.intersection(elem._from_objects) ) if self.entities: - self.entity_zero = list(self.entities)[0] + self.entity_zero = self.entities[0] elif self.namespace is not None: self.entity_zero = self.namespace else: @@ -3623,10 +3635,12 @@ class _ColumnEntity(_QueryEntity): c.entity_zero = self.entity_zero c.entities = self.entities - def setup_entity(self, ext_info, aliased_adapter): + def setup_entity(self, original_entity, ext_info, aliased_adapter): if 'selectable' not in self.__dict__: self.selectable = ext_info.selectable - self.froms.add(ext_info.selectable) + + if original_entity in self._from_entities: + self.froms.add(ext_info.selectable) def corresponds_to(self, entity): # TODO: just returning False here, diff --git a/test/orm/test_query.py b/test/orm/test_query.py index af6d960f5..8639dde74 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -3224,8 +3224,9 @@ class SessionBindTest(QueryTest): get_bind = mock.Mock(side_effect=session.get_bind) with mock.patch.object(session, "get_bind", get_bind): yield - is_(get_bind.mock_calls[0][1][0], inspect(self.classes.User)) - is_not_(get_bind.mock_calls[0][2]['clause'], None) + for call_ in get_bind.mock_calls: + is_(call_[1][0], inspect(self.classes.User)) + is_not_(call_[2]['clause'], None) def test_single_entity_q(self): User = self.classes.User @@ -3251,20 +3252,34 @@ class SessionBindTest(QueryTest): with self._assert_bind_args(session): session.query(func.max(User.name)).all() - def test_bulk_update(self): + def test_bulk_update_no_sync(self): User = self.classes.User session = Session() with self._assert_bind_args(session): session.query(User).filter(User.id == 15).update( {"name": "foob"}, synchronize_session=False) - def test_bulk_delete(self): + def test_bulk_delete_no_sync(self): User = self.classes.User session = Session() with self._assert_bind_args(session): session.query(User).filter(User.id == 15).delete( synchronize_session=False) + def test_bulk_update_fetch_sync(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).filter(User.id == 15).update( + {"name": "foob"}, synchronize_session='fetch') + + def test_bulk_delete_fetch_sync(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).filter(User.id == 15).delete( + synchronize_session='fetch') + def test_column_property(self): User = self.classes.User -- cgit v1.2.1 From 26a1d8e77c26d69cdca6e6a41ed7c3526bf27495 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 19 Jan 2015 18:00:21 -0500 Subject: - reverse the last commit temporarily as it breaks all the polymorphic cases --- doc/build/changelog/changelog_10.rst | 17 ------------ doc/build/changelog/migration_10.rst | 53 ----------------------------------- lib/sqlalchemy/orm/persistence.py | 12 +++----- lib/sqlalchemy/orm/query.py | 54 +++++++++++++----------------------- test/orm/test_query.py | 23 +++------------ 5 files changed, 28 insertions(+), 131 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 79e43e6a3..089c9fafb 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,23 +22,6 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. - .. change:: - :tags: bug, orm - :tickets: 3227, 3242, 1326 - - The primary :class:`.Mapper` of a :class:`.Query` is now passed to the - :meth:`.Session.get_bind` method when calling upon - :meth:`.Query.count`, :meth:`.Query.update`, :meth:`.Query.delete`, - as well as queries against mapped columns, - :obj:`.column_property` objects, and SQL functions and expressions - derived from mapped columns. This allows sessions that rely upon - either customized :meth:`.Session.get_bind` schemes or "bound" metadata - to work in all relevant cases. - - .. seealso:: - - :ref:`bug_3227` - .. change:: :tags: enhancement, sql :tickets: 3074 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index c0369d8b8..bd878f4cb 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -381,59 +381,6 @@ of inheritance-oriented scenarios, including: :ticket:`3035` - -.. _bug_3227: - -Session.get_bind() will receive the Mapper in all relevant Query cases ------------------------------------------------------------------------ - -A series of issues were repaired where the :meth:`.Session.get_bind` -would not receive the primary :class:`.Mapper` of the :class:`.Query`, -even though this mapper was readily available (the primary mapper is the -single mapper, or alternatively the first mapper, that is associated with -a :class:`.Query` object). - -The :class:`.Mapper` object, when passed to :meth:`.Session.get_bind`, -is typically used by sessions that make use of the -:paramref:`.Session.binds` parameter to associate mappers with a -series of engines (although in this use case, things frequently -"worked" in most cases anyway as the bind would be located via the -mapped table object), or more specifically implement a user-defined -:meth:`.Session.get_bind` method that provies some pattern of -selecting engines based on mappers, such as horizontal sharding or a -so-called "routing" session that routes queries to different backends. - -These scenarios include: - -* :meth:`.Query.count`:: - - session.query(User).count() - -* :meth:`.Query.update` and :meth:`.Query.delete`, both for the UPDATE/DELETE - statement as well as for the SELECT used by the "fetch" strategy:: - - session.query(User).filter(User.id == 15).update( - {"name": "foob"}, synchronize_session='fetch') - - session.query(User).filter(User.id == 15).delete( - synchronize_session='fetch') - -* Queries against individual columns:: - - session.query(User.id, User.name).all() - -* SQL functions and other expressions against indirect mappings such as - :obj:`.column_property`:: - - class User(Base): - # ... - - score = column_property(func.coalesce(self.tables.users.c.name, None))) - - session.query(func.max(User.score)).scalar() - -:ticket:`3227` :ticket:`3242` :ticket:`1326` - .. _feature_2963: .info dictionary improvements diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index c3b2d7bcb..e553f399d 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -1030,7 +1030,6 @@ class BulkUD(object): def __init__(self, query): self.query = query.enable_eagerloads(False) - self.mapper = self.query._bind_mapper() @property def session(self): @@ -1125,7 +1124,6 @@ class BulkFetch(BulkUD): self.primary_table.primary_key) self.matched_rows = session.execute( select_stmt, - mapper=self.mapper, params=query._params).fetchall() @@ -1136,6 +1134,7 @@ class BulkUpdate(BulkUD): super(BulkUpdate, self).__init__(query) self.query._no_select_modifiers("update") self.values = values + self.mapper = self.query._mapper_zero_or_none() @classmethod def factory(cls, query, synchronize_session, values): @@ -1181,8 +1180,7 @@ class BulkUpdate(BulkUD): self.context.whereclause, values) self.result = self.query.session.execute( - update_stmt, params=self.query._params, - mapper=self.mapper) + update_stmt, params=self.query._params) self.rowcount = self.result.rowcount def _do_post(self): @@ -1209,10 +1207,8 @@ class BulkDelete(BulkUD): delete_stmt = sql.delete(self.primary_table, self.context.whereclause) - self.result = self.query.session.execute( - delete_stmt, - params=self.query._params, - mapper=self.mapper) + self.result = self.query.session.execute(delete_stmt, + params=self.query._params) self.rowcount = self.result.rowcount def _do_post(self): diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index cd8b0efbe..7302574e6 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -146,7 +146,7 @@ class Query(object): ext_info, aliased_adapter ) - ent.setup_entity(ent, *d[entity]) + ent.setup_entity(*d[entity]) def _mapper_loads_polymorphically_with(self, mapper, adapter): for m2 in mapper._with_polymorphic_mappers or [mapper]: @@ -160,6 +160,7 @@ class Query(object): for from_obj in obj: info = inspect(from_obj) + if hasattr(info, 'mapper') and \ (info.is_mapper or info.is_aliased_class): self._select_from_entity = from_obj @@ -285,9 +286,8 @@ class Query(object): return self._entities[0] def _mapper_zero(self): - return self._select_from_entity \ - if self._select_from_entity is not None \ - else self._entity_zero().entity_zero + return self._select_from_entity or \ + self._entity_zero().entity_zero @property def _mapper_entities(self): @@ -301,14 +301,11 @@ class Query(object): self._mapper_zero() ) - def _bind_mapper(self): - ezero = self._mapper_zero() - if ezero is not None: - insp = inspect(ezero) - if hasattr(insp, 'mapper'): - return insp.mapper - - return None + def _mapper_zero_or_none(self): + if self._primary_entity: + return self._primary_entity.mapper + else: + return None def _only_mapper_zero(self, rationale=None): if len(self._entities) > 1: @@ -991,7 +988,6 @@ class Query(object): statement.correlate(None) q = self._from_selectable(fromclause) q._enable_single_crit = False - q._select_from_entity = self._mapper_zero() if entities: q._set_entities(entities) return q @@ -2530,7 +2526,7 @@ class Query(object): def _execute_and_instances(self, querycontext): conn = self._connection_from_session( - mapper=self._bind_mapper(), + mapper=self._mapper_zero_or_none(), clause=querycontext.statement, close_with_result=True) @@ -3164,7 +3160,7 @@ class _MapperEntity(_QueryEntity): supports_single_entity = True - def setup_entity(self, original_entity, ext_info, aliased_adapter): + def setup_entity(self, ext_info, aliased_adapter): self.mapper = ext_info.mapper self.aliased_adapter = aliased_adapter self.selectable = ext_info.selectable @@ -3511,9 +3507,9 @@ class _BundleEntity(_QueryEntity): for ent in self._entities: ent.adapt_to_selectable(c, sel) - def setup_entity(self, original_entity, ext_info, aliased_adapter): + def setup_entity(self, ext_info, aliased_adapter): for ent in self._entities: - ent.setup_entity(original_entity, ext_info, aliased_adapter) + ent.setup_entity(ext_info, aliased_adapter) def setup_context(self, query, context): for ent in self._entities: @@ -3596,23 +3592,15 @@ class _ColumnEntity(_QueryEntity): # leaking out their entities into the main select construct self.actual_froms = actual_froms = set(column._from_objects) - all_elements = [ - elem for elem in visitors.iterate(column, {}) - if 'parententity' in elem._annotations - ] - - self.entities = util.unique_list([ - elem._annotations['parententity'] - for elem in all_elements - ]) - self._from_entities = set( + self.entities = util.OrderedSet( elem._annotations['parententity'] - for elem in all_elements - if actual_froms.intersection(elem._from_objects) + for elem in visitors.iterate(column, {}) + if 'parententity' in elem._annotations + and actual_froms.intersection(elem._from_objects) ) if self.entities: - self.entity_zero = self.entities[0] + self.entity_zero = list(self.entities)[0] elif self.namespace is not None: self.entity_zero = self.namespace else: @@ -3635,12 +3623,10 @@ class _ColumnEntity(_QueryEntity): c.entity_zero = self.entity_zero c.entities = self.entities - def setup_entity(self, original_entity, ext_info, aliased_adapter): + def setup_entity(self, ext_info, aliased_adapter): if 'selectable' not in self.__dict__: self.selectable = ext_info.selectable - - if original_entity in self._from_entities: - self.froms.add(ext_info.selectable) + self.froms.add(ext_info.selectable) def corresponds_to(self, entity): # TODO: just returning False here, diff --git a/test/orm/test_query.py b/test/orm/test_query.py index 8639dde74..af6d960f5 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -3224,9 +3224,8 @@ class SessionBindTest(QueryTest): get_bind = mock.Mock(side_effect=session.get_bind) with mock.patch.object(session, "get_bind", get_bind): yield - for call_ in get_bind.mock_calls: - is_(call_[1][0], inspect(self.classes.User)) - is_not_(call_[2]['clause'], None) + is_(get_bind.mock_calls[0][1][0], inspect(self.classes.User)) + is_not_(get_bind.mock_calls[0][2]['clause'], None) def test_single_entity_q(self): User = self.classes.User @@ -3252,34 +3251,20 @@ class SessionBindTest(QueryTest): with self._assert_bind_args(session): session.query(func.max(User.name)).all() - def test_bulk_update_no_sync(self): + def test_bulk_update(self): User = self.classes.User session = Session() with self._assert_bind_args(session): session.query(User).filter(User.id == 15).update( {"name": "foob"}, synchronize_session=False) - def test_bulk_delete_no_sync(self): + def test_bulk_delete(self): User = self.classes.User session = Session() with self._assert_bind_args(session): session.query(User).filter(User.id == 15).delete( synchronize_session=False) - def test_bulk_update_fetch_sync(self): - User = self.classes.User - session = Session() - with self._assert_bind_args(session): - session.query(User).filter(User.id == 15).update( - {"name": "foob"}, synchronize_session='fetch') - - def test_bulk_delete_fetch_sync(self): - User = self.classes.User - session = Session() - with self._assert_bind_args(session): - session.query(User).filter(User.id == 15).delete( - synchronize_session='fetch') - def test_column_property(self): User = self.classes.User -- cgit v1.2.1 From 234a5b9723fbbb9747c0f9e3917baf8500b73370 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 19 Jan 2015 18:31:10 -0500 Subject: - restore r611883ffb35ca6664649f6328ae8 with additional fixes and an additional test that is much more specific to #1326 --- doc/build/changelog/changelog_10.rst | 17 ++++++++++++ doc/build/changelog/migration_10.rst | 53 ++++++++++++++++++++++++++++++++++++ lib/sqlalchemy/orm/persistence.py | 12 +++++--- lib/sqlalchemy/orm/query.py | 43 ++++++++++++++++++++--------- test/orm/test_query.py | 41 +++++++++++++++++++++++++--- 5 files changed, 145 insertions(+), 21 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 089c9fafb..79e43e6a3 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,23 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: bug, orm + :tickets: 3227, 3242, 1326 + + The primary :class:`.Mapper` of a :class:`.Query` is now passed to the + :meth:`.Session.get_bind` method when calling upon + :meth:`.Query.count`, :meth:`.Query.update`, :meth:`.Query.delete`, + as well as queries against mapped columns, + :obj:`.column_property` objects, and SQL functions and expressions + derived from mapped columns. This allows sessions that rely upon + either customized :meth:`.Session.get_bind` schemes or "bound" metadata + to work in all relevant cases. + + .. seealso:: + + :ref:`bug_3227` + .. change:: :tags: enhancement, sql :tickets: 3074 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index bd878f4cb..c0369d8b8 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -381,6 +381,59 @@ of inheritance-oriented scenarios, including: :ticket:`3035` + +.. _bug_3227: + +Session.get_bind() will receive the Mapper in all relevant Query cases +----------------------------------------------------------------------- + +A series of issues were repaired where the :meth:`.Session.get_bind` +would not receive the primary :class:`.Mapper` of the :class:`.Query`, +even though this mapper was readily available (the primary mapper is the +single mapper, or alternatively the first mapper, that is associated with +a :class:`.Query` object). + +The :class:`.Mapper` object, when passed to :meth:`.Session.get_bind`, +is typically used by sessions that make use of the +:paramref:`.Session.binds` parameter to associate mappers with a +series of engines (although in this use case, things frequently +"worked" in most cases anyway as the bind would be located via the +mapped table object), or more specifically implement a user-defined +:meth:`.Session.get_bind` method that provies some pattern of +selecting engines based on mappers, such as horizontal sharding or a +so-called "routing" session that routes queries to different backends. + +These scenarios include: + +* :meth:`.Query.count`:: + + session.query(User).count() + +* :meth:`.Query.update` and :meth:`.Query.delete`, both for the UPDATE/DELETE + statement as well as for the SELECT used by the "fetch" strategy:: + + session.query(User).filter(User.id == 15).update( + {"name": "foob"}, synchronize_session='fetch') + + session.query(User).filter(User.id == 15).delete( + synchronize_session='fetch') + +* Queries against individual columns:: + + session.query(User.id, User.name).all() + +* SQL functions and other expressions against indirect mappings such as + :obj:`.column_property`:: + + class User(Base): + # ... + + score = column_property(func.coalesce(self.tables.users.c.name, None))) + + session.query(func.max(User.score)).scalar() + +:ticket:`3227` :ticket:`3242` :ticket:`1326` + .. _feature_2963: .info dictionary improvements diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index e553f399d..c3b2d7bcb 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -1030,6 +1030,7 @@ class BulkUD(object): def __init__(self, query): self.query = query.enable_eagerloads(False) + self.mapper = self.query._bind_mapper() @property def session(self): @@ -1124,6 +1125,7 @@ class BulkFetch(BulkUD): self.primary_table.primary_key) self.matched_rows = session.execute( select_stmt, + mapper=self.mapper, params=query._params).fetchall() @@ -1134,7 +1136,6 @@ class BulkUpdate(BulkUD): super(BulkUpdate, self).__init__(query) self.query._no_select_modifiers("update") self.values = values - self.mapper = self.query._mapper_zero_or_none() @classmethod def factory(cls, query, synchronize_session, values): @@ -1180,7 +1181,8 @@ class BulkUpdate(BulkUD): self.context.whereclause, values) self.result = self.query.session.execute( - update_stmt, params=self.query._params) + update_stmt, params=self.query._params, + mapper=self.mapper) self.rowcount = self.result.rowcount def _do_post(self): @@ -1207,8 +1209,10 @@ class BulkDelete(BulkUD): delete_stmt = sql.delete(self.primary_table, self.context.whereclause) - self.result = self.query.session.execute(delete_stmt, - params=self.query._params) + self.result = self.query.session.execute( + delete_stmt, + params=self.query._params, + mapper=self.mapper) self.rowcount = self.result.rowcount def _do_post(self): diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 7302574e6..60a637952 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -160,7 +160,6 @@ class Query(object): for from_obj in obj: info = inspect(from_obj) - if hasattr(info, 'mapper') and \ (info.is_mapper or info.is_aliased_class): self._select_from_entity = from_obj @@ -286,8 +285,9 @@ class Query(object): return self._entities[0] def _mapper_zero(self): - return self._select_from_entity or \ - self._entity_zero().entity_zero + return self._select_from_entity \ + if self._select_from_entity is not None \ + else self._entity_zero().entity_zero @property def _mapper_entities(self): @@ -301,11 +301,14 @@ class Query(object): self._mapper_zero() ) - def _mapper_zero_or_none(self): - if self._primary_entity: - return self._primary_entity.mapper - else: - return None + def _bind_mapper(self): + ezero = self._mapper_zero() + if ezero is not None: + insp = inspect(ezero) + if hasattr(insp, 'mapper'): + return insp.mapper + + return None def _only_mapper_zero(self, rationale=None): if len(self._entities) > 1: @@ -988,6 +991,7 @@ class Query(object): statement.correlate(None) q = self._from_selectable(fromclause) q._enable_single_crit = False + q._select_from_entity = self._mapper_zero() if entities: q._set_entities(entities) return q @@ -2526,7 +2530,7 @@ class Query(object): def _execute_and_instances(self, querycontext): conn = self._connection_from_session( - mapper=self._mapper_zero_or_none(), + mapper=self._bind_mapper(), clause=querycontext.statement, close_with_result=True) @@ -3592,15 +3596,26 @@ class _ColumnEntity(_QueryEntity): # leaking out their entities into the main select construct self.actual_froms = actual_froms = set(column._from_objects) - self.entities = util.OrderedSet( + all_elements = [ + elem for elem in visitors.iterate(column, {}) + if 'parententity' in elem._annotations + ] + + self.entities = util.unique_list( + elem._annotations['parententity'] + for elem in all_elements + if 'parententity' in elem._annotations + ) + + self._from_entities = set( elem._annotations['parententity'] - for elem in visitors.iterate(column, {}) + for elem in all_elements if 'parententity' in elem._annotations and actual_froms.intersection(elem._from_objects) ) if self.entities: - self.entity_zero = list(self.entities)[0] + self.entity_zero = self.entities[0] elif self.namespace is not None: self.entity_zero = self.namespace else: @@ -3626,7 +3641,9 @@ class _ColumnEntity(_QueryEntity): def setup_entity(self, ext_info, aliased_adapter): if 'selectable' not in self.__dict__: self.selectable = ext_info.selectable - self.froms.add(ext_info.selectable) + + if self.actual_froms.intersection(ext_info.selectable._from_objects): + self.froms.add(ext_info.selectable) def corresponds_to(self, entity): # TODO: just returning False here, diff --git a/test/orm/test_query.py b/test/orm/test_query.py index af6d960f5..a2a1ee096 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -3224,8 +3224,9 @@ class SessionBindTest(QueryTest): get_bind = mock.Mock(side_effect=session.get_bind) with mock.patch.object(session, "get_bind", get_bind): yield - is_(get_bind.mock_calls[0][1][0], inspect(self.classes.User)) - is_not_(get_bind.mock_calls[0][2]['clause'], None) + for call_ in get_bind.mock_calls: + is_(call_[1][0], inspect(self.classes.User)) + is_not_(call_[2]['clause'], None) def test_single_entity_q(self): User = self.classes.User @@ -3251,20 +3252,34 @@ class SessionBindTest(QueryTest): with self._assert_bind_args(session): session.query(func.max(User.name)).all() - def test_bulk_update(self): + def test_bulk_update_no_sync(self): User = self.classes.User session = Session() with self._assert_bind_args(session): session.query(User).filter(User.id == 15).update( {"name": "foob"}, synchronize_session=False) - def test_bulk_delete(self): + def test_bulk_delete_no_sync(self): User = self.classes.User session = Session() with self._assert_bind_args(session): session.query(User).filter(User.id == 15).delete( synchronize_session=False) + def test_bulk_update_fetch_sync(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).filter(User.id == 15).update( + {"name": "foob"}, synchronize_session='fetch') + + def test_bulk_delete_fetch_sync(self): + User = self.classes.User + session = Session() + with self._assert_bind_args(session): + session.query(User).filter(User.id == 15).delete( + synchronize_session='fetch') + def test_column_property(self): User = self.classes.User @@ -3275,3 +3290,21 @@ class SessionBindTest(QueryTest): session = Session() with self._assert_bind_args(session): session.query(func.max(User.score)).scalar() + + def test_column_property_select(self): + User = self.classes.User + Address = self.classes.Address + + mapper = inspect(User) + mapper.add_property( + "score", + column_property( + select([func.sum(Address.id)]). + where(Address.user_id == User.id).as_scalar() + ) + ) + session = Session() + + with self._assert_bind_args(session): + session.query(func.max(User.score)).scalar() + -- cgit v1.2.1 From 10dd5fe81062347905492ef66e6f0453479cc03b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 20 Jan 2015 11:03:02 -0500 Subject: formatting --- test/engine/test_transaction.py | 57 ++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/test/engine/test_transaction.py b/test/engine/test_transaction.py index b3b17e75a..b7ad01408 100644 --- a/test/engine/test_transaction.py +++ b/test/engine/test_transaction.py @@ -1240,7 +1240,7 @@ class IsolationLevelTest(fixtures.TestBase): eng = testing_engine() isolation_level = eng.dialect.get_isolation_level( - eng.connect().connection) + eng.connect().connection) level = self._non_default_isolation_level() ne_(isolation_level, level) @@ -1248,7 +1248,7 @@ class IsolationLevelTest(fixtures.TestBase): eng = testing_engine(options=dict(isolation_level=level)) eq_( eng.dialect.get_isolation_level( - eng.connect().connection), + eng.connect().connection), level ) @@ -1270,7 +1270,7 @@ class IsolationLevelTest(fixtures.TestBase): def test_default_level(self): eng = testing_engine(options=dict()) isolation_level = eng.dialect.get_isolation_level( - eng.connect().connection) + eng.connect().connection) eq_(isolation_level, self._default_isolation_level()) def test_reset_level(self): @@ -1282,8 +1282,8 @@ class IsolationLevelTest(fixtures.TestBase): ) eng.dialect.set_isolation_level( - conn.connection, self._non_default_isolation_level() - ) + conn.connection, self._non_default_isolation_level() + ) eq_( eng.dialect.get_isolation_level(conn.connection), self._non_default_isolation_level() @@ -1298,14 +1298,15 @@ class IsolationLevelTest(fixtures.TestBase): conn.close() def test_reset_level_with_setting(self): - eng = testing_engine(options=dict( - isolation_level= - self._non_default_isolation_level())) + eng = testing_engine( + options=dict( + isolation_level=self._non_default_isolation_level())) conn = eng.connect() eq_(eng.dialect.get_isolation_level(conn.connection), self._non_default_isolation_level()) - eng.dialect.set_isolation_level(conn.connection, - self._default_isolation_level()) + eng.dialect.set_isolation_level( + conn.connection, + self._default_isolation_level()) eq_(eng.dialect.get_isolation_level(conn.connection), self._default_isolation_level()) eng.dialect.reset_isolation_level(conn.connection) @@ -1317,22 +1318,24 @@ class IsolationLevelTest(fixtures.TestBase): eng = testing_engine(options=dict(isolation_level='FOO')) assert_raises_message( exc.ArgumentError, - "Invalid value '%s' for isolation_level. " - "Valid isolation levels for %s are %s" % - ("FOO", eng.dialect.name, - ", ".join(eng.dialect._isolation_lookup)), - eng.connect) + "Invalid value '%s' for isolation_level. " + "Valid isolation levels for %s are %s" % + ("FOO", + eng.dialect.name, ", ".join(eng.dialect._isolation_lookup)), + eng.connect + ) def test_per_connection(self): from sqlalchemy.pool import QueuePool - eng = testing_engine(options=dict( - poolclass=QueuePool, - pool_size=2, max_overflow=0)) + eng = testing_engine( + options=dict( + poolclass=QueuePool, + pool_size=2, max_overflow=0)) c1 = eng.connect() c1 = c1.execution_options( - isolation_level=self._non_default_isolation_level() - ) + isolation_level=self._non_default_isolation_level() + ) c2 = eng.connect() eq_( eng.dialect.get_isolation_level(c1.connection), @@ -1366,17 +1369,17 @@ class IsolationLevelTest(fixtures.TestBase): r"per-engine using the isolation_level " r"argument to create_engine\(\).", select([1]).execution_options, - isolation_level=self._non_default_isolation_level() + isolation_level=self._non_default_isolation_level() ) - def test_per_engine(self): # new in 0.9 - eng = create_engine(testing.db.url, - execution_options={ - 'isolation_level': - self._non_default_isolation_level()} - ) + eng = create_engine( + testing.db.url, + execution_options={ + 'isolation_level': + self._non_default_isolation_level()} + ) conn = eng.connect() eq_( eng.dialect.get_isolation_level(conn.connection), -- cgit v1.2.1 From 4032aaf097a9268bc331e4b4815d77b19ba3febb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 20 Jan 2015 11:36:14 -0500 Subject: - enhance detail here regarding the difference between Connection.connection and engine.raw_connection() --- doc/build/core/connections.rst | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/doc/build/core/connections.rst b/doc/build/core/connections.rst index 5f0bb4972..6d7e7622f 100644 --- a/doc/build/core/connections.rst +++ b/doc/build/core/connections.rst @@ -453,13 +453,36 @@ Working with Raw DBAPI Connections There are some cases where SQLAlchemy does not provide a genericized way at accessing some :term:`DBAPI` functions, such as calling stored procedures as well as dealing with multiple result sets. In these cases, it's just as expedient -to deal with the raw DBAPI connection directly. This is accessible from -a :class:`.Engine` using the :meth:`.Engine.raw_connection` method:: +to deal with the raw DBAPI connection directly. + +The most common way to access the raw DBAPI connection is to get it +from an already present :class:`.Connection` object directly. It is +present using the :attr:`.Connection.connection` attribute:: + + connection = engine.connect() + dbapi_conn = connection.connection + +The DBAPI connection here is actually a "proxied" in terms of the +originating connection pool, however this is an implementation detail +that in most cases can be ignored. As this DBAPI connection is still +contained within the scope of an owning :class:`.Connection` object, it is +best to make use of the :class:`.Connection` object for most features such +as transaction control as well as calling the :meth:`.Connection.close` +method; if these operations are performed on the DBAPI connection directly, +the owning :class:`.Connection` will not be aware of these changes in state. + +To overcome the limitations imposed by the DBAPI connection that is +maintained by an owning :class:`.Connection`, a DBAPI connection is also +available without the need to procure a +:class:`.Connection` first, using the :meth:`.Engine.raw_connection` method +of :class:`.Engine`:: dbapi_conn = engine.raw_connection() -The instance returned is a "wrapped" form of DBAPI connection. When its -``.close()`` method is called, the connection is :term:`released` back to the +This DBAPI connection is again a "proxied" form as was the case before. +The purpose of this proxying is now apparent, as when we call the ``.close()`` +method of this connection, the DBAPI connection is typically not actually +closed, but instead :term:`released` back to the engine's connection pool:: dbapi_conn.close() -- cgit v1.2.1 From c3d898e8d06c7e549bb273fc8654f5d24fab2204 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 20 Jan 2015 11:37:13 -0500 Subject: - Added new user-space accessors for viewing transaction isolation levels; :meth:`.Connection.get_isolation_level`, :attr:`.Connection.default_isolation_level`. - enhance documentation inter-linkage between new accessors, existing isolation_level parameters, as well as in the dialect-level methods which should be fully covered by Engine/Connection level APIs now. --- doc/build/changelog/changelog_09.rst | 8 ++ lib/sqlalchemy/dialects/mysql/base.py | 2 +- lib/sqlalchemy/dialects/postgresql/base.py | 2 +- lib/sqlalchemy/dialects/sqlite/base.py | 2 + lib/sqlalchemy/engine/__init__.py | 12 ++- lib/sqlalchemy/engine/base.py | 118 ++++++++++++++++++++++++++--- lib/sqlalchemy/engine/interfaces.py | 71 ++++++++++++++++- test/engine/test_execute.py | 21 +++++ test/engine/test_transaction.py | 22 ++++++ 9 files changed, 241 insertions(+), 17 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index d9cbd5032..b1ec9cbec 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -14,6 +14,14 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: feature, engine + :versions: 1.0.0 + + Added new user-space accessors for viewing transaction isolation + levels; :meth:`.Connection.get_isolation_level`, + :attr:`.Connection.default_isolation_level`. + .. change:: :tags: bug, postgresql :versions: 1.0.0 diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index ca56a4d23..c8e33bfb2 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -106,7 +106,7 @@ to be used. Transaction Isolation Level --------------------------- -:func:`.create_engine` accepts an ``isolation_level`` +:func:`.create_engine` accepts an :paramref:`.create_engine.isolation_level` parameter which results in the command ``SET SESSION TRANSACTION ISOLATION LEVEL `` being invoked for every new connection. Valid values for this parameter are diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 89bea100e..1935d0cad 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -48,7 +48,7 @@ Transaction Isolation Level --------------------------- All Postgresql dialects support setting of transaction isolation level -both via a dialect-specific parameter ``isolation_level`` +both via a dialect-specific parameter :paramref:`.create_engine.isolation_level` accepted by :func:`.create_engine`, as well as the ``isolation_level`` argument as passed to :meth:`.Connection.execution_options`. When using a non-psycopg2 dialect, diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index f74421967..1ed89bacb 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -107,6 +107,8 @@ The following subsections introduce areas that are impacted by SQLite's file-based architecture and additionally will usually require workarounds to work when using the pysqlite driver. +.. _sqlite_isolation_level: + Transaction Isolation Level ---------------------------- diff --git a/lib/sqlalchemy/engine/__init__.py b/lib/sqlalchemy/engine/__init__.py index 3857bdf1e..f512e260a 100644 --- a/lib/sqlalchemy/engine/__init__.py +++ b/lib/sqlalchemy/engine/__init__.py @@ -257,9 +257,19 @@ def create_engine(*args, **kwargs): Behavior here varies per backend, and individual dialects should be consulted directly. + Note that the isolation level can also be set on a per-:class:`.Connection` + basis as well, using the + :paramref:`.Connection.execution_options.isolation_level` + feature. + .. seealso:: - :ref:`SQLite Concurrency ` + :attr:`.Connection.default_isolation_level` - view default level + + :paramref:`.Connection.execution_options.isolation_level` + - set per :class:`.Connection` isolation level + + :ref:`SQLite Transaction Isolation ` :ref:`Postgresql Transaction Isolation ` diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index ee8267c5c..fa5dfca9a 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -201,14 +201,19 @@ class Connection(Connectable): used by the ORM internally supersedes a cache dictionary specified here. - :param isolation_level: Available on: Connection. + :param isolation_level: Available on: :class:`.Connection`. Set the transaction isolation level for - the lifespan of this connection. Valid values include - those string values accepted by the ``isolation_level`` - parameter passed to :func:`.create_engine`, and are - database specific, including those for :ref:`sqlite_toplevel`, - :ref:`postgresql_toplevel` - see those dialect's documentation - for further info. + the lifespan of this :class:`.Connection` object (*not* the + underyling DBAPI connection, for which the level is reset + to its original setting upon termination of this + :class:`.Connection` object). + + Valid values include + those string values accepted by the + :paramref:`.create_engine.isolation_level` + parameter passed to :func:`.create_engine`. These levels are + semi-database specific; see individual dialect documentation for + valid levels. Note that this option necessarily affects the underlying DBAPI connection for the lifespan of the originating @@ -217,6 +222,20 @@ class Connection(Connectable): is returned to the connection pool, i.e. the :meth:`.Connection.close` method is called. + .. seealso:: + + :paramref:`.create_engine.isolation_level` + - set per :class:`.Engine` isolation level + + :meth:`.Connection.get_isolation_level` - view current level + + :ref:`SQLite Transaction Isolation ` + + :ref:`Postgresql Transaction Isolation ` + + :ref:`MySQL Transaction Isolation ` + + :param no_parameters: When ``True``, if the final parameter list or dictionary is totally empty, will invoke the statement on the cursor as ``cursor.execute(statement)``, @@ -260,7 +279,14 @@ class Connection(Connectable): @property def connection(self): - "The underlying DB-API connection managed by this Connection." + """The underlying DB-API connection managed by this Connection. + + .. seealso:: + + + :ref:`dbapi_connections` + + """ try: return self.__connection @@ -270,6 +296,71 @@ class Connection(Connectable): except Exception as e: self._handle_dbapi_exception(e, None, None, None, None) + def get_isolation_level(self): + """Return the current isolation level assigned to this + :class:`.Connection`. + + This will typically be the default isolation level as determined + by the dialect, unless if the + :paramref:`.Connection.execution_options.isolation_level` + feature has been used to alter the isolation level on a + per-:class:`.Connection` basis. + + This attribute will typically perform a live SQL operation in order + to procure the current isolation level, so the value returned is the + actual level on the underlying DBAPI connection regardless of how + this state was set. Compare to the + :attr:`.Connection.default_isolation_level` accessor + which returns the dialect-level setting without performing a SQL + query. + + .. versionadded:: 0.9.9 + + .. seealso:: + + :attr:`.Connection.default_isolation_level` - view default level + + :paramref:`.create_engine.isolation_level` + - set per :class:`.Engine` isolation level + + :paramref:`.Connection.execution_options.isolation_level` + - set per :class:`.Connection` isolation level + + """ + try: + return self.dialect.get_isolation_level(self.connection) + except Exception as e: + self._handle_dbapi_exception(e, None, None, None, None) + + @property + def default_isolation_level(self): + """The default isolation level assigned to this :class:`.Connection`. + + This is the isolation level setting that the :class:`.Connection` + has when first procured via the :meth:`.Engine.connect` method. + This level stays in place until the + :paramref:`.Connection.execution_options.isolation_level` is used + to change the setting on a per-:class:`.Connection` basis. + + Unlike :meth:`.Connection.get_isolation_level`, this attribute is set + ahead of time from the first connection procured by the dialect, + so SQL query is not invoked when this accessor is called. + + .. versionadded:: 0.9.9 + + .. seealso:: + + :meth:`.Connection.get_isolation_level` - view current level + + :paramref:`.create_engine.isolation_level` + - set per :class:`.Engine` isolation level + + :paramref:`.Connection.execution_options.isolation_level` + - set per :class:`.Connection` isolation level + + """ + return self.dialect.default_isolation_level + def _revalidate_connection(self): if self.__branch_from: return self.__branch_from._revalidate_connection() @@ -1982,9 +2073,14 @@ class Engine(Connectable, log.Identified): for real. This method provides direct DBAPI connection access for - special situations. In most situations, the :class:`.Connection` - object should be used, which is procured using the - :meth:`.Engine.connect` method. + special situations when the API provided by :class:`.Connection` + is not needed. When a :class:`.Connection` object is already + present, the DBAPI connection is available using + the :attr:`.Connection.connection` accessor. + + .. seealso:: + + :ref:`dbapi_connections` """ return self._wrap_pool_connect( diff --git a/lib/sqlalchemy/engine/interfaces.py b/lib/sqlalchemy/engine/interfaces.py index 5f66e54b5..5f0d74328 100644 --- a/lib/sqlalchemy/engine/interfaces.py +++ b/lib/sqlalchemy/engine/interfaces.py @@ -654,17 +654,82 @@ class Dialect(object): return None def reset_isolation_level(self, dbapi_conn): - """Given a DBAPI connection, revert its isolation to the default.""" + """Given a DBAPI connection, revert its isolation to the default. + + Note that this is a dialect-level method which is used as part + of the implementation of the :class:`.Connection` and + :class:`.Engine` + isolation level facilities; these APIs should be preferred for + most typical use cases. + + .. seealso:: + + :meth:`.Connection.get_isolation_level` - view current level + + :attr:`.Connection.default_isolation_level` - view default level + + :paramref:`.Connection.execution_options.isolation_level` - + set per :class:`.Connection` isolation level + + :paramref:`.create_engine.isolation_level` - + set per :class:`.Engine` isolation level + + """ raise NotImplementedError() def set_isolation_level(self, dbapi_conn, level): - """Given a DBAPI connection, set its isolation level.""" + """Given a DBAPI connection, set its isolation level. + + Note that this is a dialect-level method which is used as part + of the implementation of the :class:`.Connection` and + :class:`.Engine` + isolation level facilities; these APIs should be preferred for + most typical use cases. + + .. seealso:: + + :meth:`.Connection.get_isolation_level` - view current level + + :attr:`.Connection.default_isolation_level` - view default level + + :paramref:`.Connection.execution_options.isolation_level` - + set per :class:`.Connection` isolation level + + :paramref:`.create_engine.isolation_level` - + set per :class:`.Engine` isolation level + + """ raise NotImplementedError() def get_isolation_level(self, dbapi_conn): - """Given a DBAPI connection, return its isolation level.""" + """Given a DBAPI connection, return its isolation level. + + When working with a :class:`.Connection` object, the corresponding + DBAPI connection may be procured using the + :attr:`.Connection.connection` accessor. + + Note that this is a dialect-level method which is used as part + of the implementation of the :class:`.Connection` and + :class:`.Engine` isolation level facilities; + these APIs should be preferred for most typical use cases. + + + .. seealso:: + + :meth:`.Connection.get_isolation_level` - view current level + + :attr:`.Connection.default_isolation_level` - view default level + + :paramref:`.Connection.execution_options.isolation_level` - + set per :class:`.Connection` isolation level + + :paramref:`.create_engine.isolation_level` - + set per :class:`.Engine` isolation level + + + """ raise NotImplementedError() diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index 8e58d202d..725dcebe0 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -1900,6 +1900,27 @@ class HandleErrorTest(fixtures.TestBase): self._test_alter_disconnect(True, False) self._test_alter_disconnect(False, False) + def test_handle_error_event_connect_isolation_level(self): + engine = engines.testing_engine() + + class MySpecialException(Exception): + pass + + @event.listens_for(engine, "handle_error") + def handle_error(ctx): + raise MySpecialException("failed operation") + + ProgrammingError = engine.dialect.dbapi.ProgrammingError + with engine.connect() as conn: + with patch.object( + conn.dialect, "get_isolation_level", + Mock(side_effect=ProgrammingError("random error")) + ): + assert_raises( + MySpecialException, + conn.get_isolation_level + ) + class HandleInvalidatedOnConnectTest(fixtures.TestBase): __requires__ = ('sqlite', ) diff --git a/test/engine/test_transaction.py b/test/engine/test_transaction.py index b7ad01408..0f5bb4cb5 100644 --- a/test/engine/test_transaction.py +++ b/test/engine/test_transaction.py @@ -1385,3 +1385,25 @@ class IsolationLevelTest(fixtures.TestBase): eng.dialect.get_isolation_level(conn.connection), self._non_default_isolation_level() ) + + def test_isolation_level_accessors_connection_default(self): + eng = create_engine( + testing.db.url + ) + with eng.connect() as conn: + eq_(conn.default_isolation_level, self._default_isolation_level()) + with eng.connect() as conn: + eq_(conn.get_isolation_level(), self._default_isolation_level()) + + def test_isolation_level_accessors_connection_option_modified(self): + eng = create_engine( + testing.db.url + ) + with eng.connect() as conn: + c2 = conn.execution_options( + isolation_level=self._non_default_isolation_level()) + eq_(conn.default_isolation_level, self._default_isolation_level()) + eq_(conn.get_isolation_level(), + self._non_default_isolation_level()) + eq_(c2.get_isolation_level(), self._non_default_isolation_level()) + -- cgit v1.2.1 From a33c250da273ba9b1c62b5ba6d99914870155faf Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 25 Jan 2015 17:53:41 -0500 Subject: - remove context-specific post-crud logic from Connection and inline post-crud logic to some degree in DefaultExecutionContext. In particular we are removing post_insert() which doesn't appear to be used based on a survey of prominent third party dialects. Callcounts aren't added to existing execute profiling tests and inserts might be a little better. - simplify the execution_options join in DEC. Callcounts don't appear affected. --- lib/sqlalchemy/engine/base.py | 29 ++------ lib/sqlalchemy/engine/default.py | 127 +++++++++++++++++++++--------------- lib/sqlalchemy/util/_collections.py | 9 ++- 3 files changed, 88 insertions(+), 77 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index fa5dfca9a..aba0d29df 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -1132,31 +1132,12 @@ class Connection(Connectable): if context.compiled: context.post_exec() - if context.isinsert and not context.executemany: - context.post_insert() - - # create a resultproxy, get rowcount/implicit RETURNING - # rows, close cursor if no further results pending - result = context.get_result_proxy() - if context.isinsert: - if context._is_implicit_returning: - context._fetch_implicit_returning(result) - result.close(_autoclose_connection=False) - result._metadata = None - elif not context._is_explicit_returning: + if context.is_crud: + result = context._setup_crud_result_proxy() + else: + result = context.get_result_proxy() + if result._metadata is None: result.close(_autoclose_connection=False) - result._metadata = None - elif context.isupdate and context._is_implicit_returning: - context._fetch_implicit_update_returning(result) - result.close(_autoclose_connection=False) - result._metadata = None - - elif result._metadata is None: - # no results, get rowcount - # (which requires open cursor on some drivers - # such as kintersbasdb, mxodbc), - result.rowcount - result.close(_autoclose_connection=False) if context.should_autocommit and self._root.__transaction is None: self._root._commit_impl(autocommit=True) diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index c5b5deece..f6c2263b3 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -452,14 +452,12 @@ class DefaultExecutionContext(interfaces.ExecutionContext): isinsert = False isupdate = False isdelete = False + is_crud = False isddl = False executemany = False result_map = None compiled = None statement = None - postfetch_cols = None - prefetch_cols = None - returning_cols = None _is_implicit_returning = False _is_explicit_returning = False @@ -515,10 +513,8 @@ class DefaultExecutionContext(interfaces.ExecutionContext): if not compiled.can_execute: raise exc.ArgumentError("Not an executable clause") - self.execution_options = compiled.statement._execution_options - if connection._execution_options: - self.execution_options = dict(self.execution_options) - self.execution_options.update(connection._execution_options) + self.execution_options = compiled.statement._execution_options.union( + connection._execution_options) # compiled clauseelement. process bind params, process table defaults, # track collections used by ResultProxy to target and process results @@ -548,6 +544,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.cursor = self.create_cursor() if self.isinsert or self.isupdate or self.isdelete: + self.is_crud = True self._is_explicit_returning = bool(compiled.statement._returning) self._is_implicit_returning = bool( compiled.returning and not compiled.statement._returning) @@ -680,10 +677,6 @@ class DefaultExecutionContext(interfaces.ExecutionContext): def no_parameters(self): return self.execution_options.get("no_parameters", False) - @util.memoized_property - def is_crud(self): - return self.isinsert or self.isupdate or self.isdelete - @util.memoized_property def should_autocommit(self): autocommit = self.execution_options.get('autocommit', @@ -799,52 +792,84 @@ class DefaultExecutionContext(interfaces.ExecutionContext): def supports_sane_multi_rowcount(self): return self.dialect.supports_sane_multi_rowcount - def post_insert(self): - + def _setup_crud_result_proxy(self): + if self.isinsert and \ + not self.executemany: + if not self._is_implicit_returning and \ + not self.compiled.inline and \ + self.dialect.postfetch_lastrowid: + + self._setup_ins_pk_from_lastrowid() + + elif not self._is_implicit_returning: + self._setup_ins_pk_from_empty() + + result = self.get_result_proxy() + + if self.isinsert: + if self._is_implicit_returning: + row = result.fetchone() + self.returned_defaults = row + self._setup_ins_pk_from_implicit_returning(row) + result.close(_autoclose_connection=False) + result._metadata = None + elif not self._is_explicit_returning: + result.close(_autoclose_connection=False) + result._metadata = None + elif self.isupdate and self._is_implicit_returning: + row = result.fetchone() + self.returned_defaults = row + result.close(_autoclose_connection=False) + result._metadata = None + + elif result._metadata is None: + # no results, get rowcount + # (which requires open cursor on some drivers + # such as kintersbasdb, mxodbc) + result.rowcount + result.close(_autoclose_connection=False) + return result + + def _setup_ins_pk_from_lastrowid(self): key_getter = self.compiled._key_getters_for_crud_column[2] table = self.compiled.statement.table + compiled_params = self.compiled_parameters[0] + + lastrowid = self.get_lastrowid() + autoinc_col = table._autoincrement_column + if autoinc_col is not None: + # apply type post processors to the lastrowid + proc = autoinc_col.type._cached_result_processor( + self.dialect, None) + if proc is not None: + lastrowid = proc(lastrowid) + self.inserted_primary_key = [ + lastrowid if c is autoinc_col else + compiled_params.get(key_getter(c), None) + for c in table.primary_key + ] - if not self._is_implicit_returning and \ - not self._is_explicit_returning and \ - not self.compiled.inline and \ - self.dialect.postfetch_lastrowid: - - lastrowid = self.get_lastrowid() - autoinc_col = table._autoincrement_column - if autoinc_col is not None: - # apply type post processors to the lastrowid - proc = autoinc_col.type._cached_result_processor( - self.dialect, None) - if proc is not None: - lastrowid = proc(lastrowid) - self.inserted_primary_key = [ - lastrowid if c is autoinc_col else - self.compiled_parameters[0].get(key_getter(c), None) - for c in table.primary_key - ] - else: - self.inserted_primary_key = [ - self.compiled_parameters[0].get(key_getter(c), None) - for c in table.primary_key - ] - - def _fetch_implicit_returning(self, resultproxy): + def _setup_ins_pk_from_empty(self): + key_getter = self.compiled._key_getters_for_crud_column[2] table = self.compiled.statement.table - row = resultproxy.fetchone() - - ipk = [] - for c, v in zip(table.primary_key, self.inserted_primary_key): - if v is not None: - ipk.append(v) - else: - ipk.append(row[c]) + compiled_params = self.compiled_parameters[0] + self.inserted_primary_key = [ + compiled_params.get(key_getter(c), None) + for c in table.primary_key + ] - self.inserted_primary_key = ipk - self.returned_defaults = row + def _setup_ins_pk_from_implicit_returning(self, row): + key_getter = self.compiled._key_getters_for_crud_column[2] + table = self.compiled.statement.table + compiled_params = self.compiled_parameters[0] - def _fetch_implicit_update_returning(self, resultproxy): - row = resultproxy.fetchone() - self.returned_defaults = row + self.inserted_primary_key = [ + row[col] if value is None else value + for col, value in [ + (col, compiled_params.get(key_getter(col), None)) + for col in table.primary_key + ] + ] def lastrow_has_defaults(self): return (self.isinsert or self.isupdate) and \ diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index 0f05e3427..a49848d08 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -165,8 +165,13 @@ class immutabledict(ImmutableContainer, dict): return immutabledict, (dict(self), ) def union(self, d): - if not self: - return immutabledict(d) + if not d: + return self + elif not self: + if isinstance(d, immutabledict): + return d + else: + return immutabledict(d) else: d2 = immutabledict(self) dict.update(d2, d) -- cgit v1.2.1 From 7b102eeaee12265a2c4f4f5619827178d379d210 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 25 Jan 2015 18:22:00 -0500 Subject: - remove the clever approach w/ dialect events, and remove the need for a for-loop through an empty tuple. we add one more local flag to handle the logic without repetition of dialect.do_execute() calls. --- lib/sqlalchemy/engine/base.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index aba0d29df..8d816b7fd 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -1079,36 +1079,39 @@ class Connection(Connectable): "%r", sql_util._repr_params(parameters, batches=10) ) + + evt_handled = False try: if context.executemany: - for fn in () if not self.dialect._has_events \ - else self.dialect.dispatch.do_executemany: - if fn(cursor, statement, parameters, context): - break - else: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn(cursor, statement, parameters, context): + evt_handled = True + break + if not evt_handled: self.dialect.do_executemany( cursor, statement, parameters, context) - elif not parameters and context.no_parameters: - for fn in () if not self.dialect._has_events \ - else self.dialect.dispatch.do_execute_no_params: - if fn(cursor, statement, context): - break - else: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, statement, context): + evt_handled = True + break + if not evt_handled: self.dialect.do_execute_no_params( cursor, statement, context) - else: - for fn in () if not self.dialect._has_events \ - else self.dialect.dispatch.do_execute: - if fn(cursor, statement, parameters, context): - break - else: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn(cursor, statement, parameters, context): + evt_handled = True + break + if not evt_handled: self.dialect.do_execute( cursor, statement, -- cgit v1.2.1 From 20cdf0e8550e5d2d7a5f2b3ad1e3b8bd354e9b6c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 26 Jan 2015 16:53:59 -0500 Subject: - changelog for pullreq github:150 --- doc/build/changelog/changelog_10.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 79e43e6a3..a6a935da4 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,14 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, sql + :pullreq: github:150 + + The type of expression is reported when an object passed to a + SQL expression unit can't be interpreted as a SQL fragment; + pull request courtesy Ryan P. Kelly. + .. change:: :tags: bug, orm :tickets: 3227, 3242, 1326 -- cgit v1.2.1 From 987f40b5aa325fe8a6655bcb0be2329c0a24025d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 26 Jan 2015 17:04:40 -0500 Subject: - changelog for #3262, fixes #3262 --- doc/build/changelog/changelog_10.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index a6a935da4..18043c456 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,17 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, orm + :tickets: 3262 + :pullreq: bitbucket:38 + + A warning is emitted when the same polymorphic identity is assigned + to two different mappers in the same hierarchy. This is typically a + user error and means that the two different mapping types cannot be + correctly distinguished at load time. Pull request courtesy + Sebastian Bank. + .. change:: :tags: feature, sql :pullreq: github:150 -- cgit v1.2.1 From f94d75ede5f5d2ed28d72ff98ca7caca016e5506 Mon Sep 17 00:00:00 2001 From: Shaun Stanworth Date: Sun, 9 Nov 2014 14:52:31 +0000 Subject: Added psycopg2cffi dialect --- lib/sqlalchemy/dialects/postgresql/__init__.py | 2 +- lib/sqlalchemy/dialects/postgresql/psycopg2.py | 17 ++++++-- lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py | 46 ++++++++++++++++++++++ setup.cfg | 1 + test/dialect/postgresql/test_dialect.py | 2 +- test/dialect/postgresql/test_query.py | 1 + test/dialect/postgresql/test_types.py | 7 ++-- test/engine/test_execute.py | 2 +- test/requirements.py | 4 ++ test/sql/test_types.py | 1 + 10 files changed, 74 insertions(+), 9 deletions(-) create mode 100644 lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py diff --git a/lib/sqlalchemy/dialects/postgresql/__init__.py b/lib/sqlalchemy/dialects/postgresql/__init__.py index 1cff8e3a0..01a846314 100644 --- a/lib/sqlalchemy/dialects/postgresql/__init__.py +++ b/lib/sqlalchemy/dialects/postgresql/__init__.py @@ -5,7 +5,7 @@ # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from . import base, psycopg2, pg8000, pypostgresql, zxjdbc +from . import base, psycopg2, pg8000, pypostgresql, zxjdbc, psycopg2cffi base.dialect = psycopg2.dialect diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index fe27da8b6..5246abf1c 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -512,9 +512,19 @@ class PGDialect_psycopg2(PGDialect): import psycopg2 return psycopg2 + @classmethod + def _psycopg2_extensions(cls): + from psycopg2 import extensions + return extensions + + @classmethod + def _psycopg2_extras(cls): + from psycopg2 import extras + return extras + @util.memoized_property def _isolation_lookup(self): - from psycopg2 import extensions + extensions = self._psycopg2_extensions() return { 'AUTOCOMMIT': extensions.ISOLATION_LEVEL_AUTOCOMMIT, 'READ COMMITTED': extensions.ISOLATION_LEVEL_READ_COMMITTED, @@ -536,7 +546,8 @@ class PGDialect_psycopg2(PGDialect): connection.set_isolation_level(level) def on_connect(self): - from psycopg2 import extras, extensions + extras = self._psycopg2_extras() + extensions = self._psycopg2_extensions() fns = [] if self.client_encoding is not None: @@ -586,7 +597,7 @@ class PGDialect_psycopg2(PGDialect): @util.memoized_instancemethod def _hstore_oids(self, conn): if self.psycopg2_version >= (2, 4): - from psycopg2 import extras + extras = self._psycopg2_extras() oids = extras.HstoreAdapter.get_oids(conn) if oids is not None and oids[0]: return oids[0:2] diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py b/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py new file mode 100644 index 000000000..5217c5561 --- /dev/null +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py @@ -0,0 +1,46 @@ +# testing/engines.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +""" +.. dialect:: postgresql+psycopg2cffi + :name: psycopg2cffi + :dbapi: psycopg2cffi + :connectstring: \ +postgresql+psycopg2cffi://user:password@host:port/dbname\ +[?key=value&key=value...] + :url: http://pypi.python.org/pypi/psycopg2cffi/ + +`psycopg2cffi` is an adaptation of `psycopg2`, using CFFI for the C +layer. This makes it suitable for use in e.g. PyPy. Documentation +is as per `psycopg2`. + +.. seealso:: + + :mod:`sqlalchemy.dialects.postgresql.psycopg2` + +""" +from .psycopg2 import PGDialect_psycopg2 + + +class PGDialect_psycopg2cffi(PGDialect_psycopg2): + driver = 'psycopg2cffi' + + @classmethod + def dbapi(cls): + return __import__('psycopg2cffi') + + @classmethod + def _psycopg2_extensions(cls): + root = __import__('psycopg2cffi', fromlist=['extensions']) + return root.extensions + + @classmethod + def _psycopg2_extras(cls): + root = __import__('psycopg2cffi', fromlist=['extras']) + return root.extras + + +dialect = PGDialect_psycopg2cffi diff --git a/setup.cfg b/setup.cfg index 51a4e30bf..5eb35469f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ postgresql=postgresql://scott:tiger@127.0.0.1:5432/test pg8000=postgresql+pg8000://scott:tiger@127.0.0.1:5432/test postgres=postgresql://scott:tiger@127.0.0.1:5432/test postgresql_jython=postgresql+zxjdbc://scott:tiger@127.0.0.1:5432/test +postgresql_psycopg2cffi=postgresql+psycopg2cffi://127.0.0.1:5432/test mysql=mysql://scott:tiger@127.0.0.1:3306/test mysqlconnector=mysql+mysqlconnector://scott:tiger@127.0.0.1:3306/test mssql=mssql+pyodbc://scott:tiger@ms_2008 diff --git a/test/dialect/postgresql/test_dialect.py b/test/dialect/postgresql/test_dialect.py index b751bbcdd..2166bca32 100644 --- a/test/dialect/postgresql/test_dialect.py +++ b/test/dialect/postgresql/test_dialect.py @@ -118,7 +118,7 @@ class MiscTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL): eq_(c.connection.connection.encoding, test_encoding) @testing.only_on( - ['postgresql+psycopg2', 'postgresql+pg8000'], + ['postgresql+psycopg2', 'postgresql+pg8000', 'postgresql+psycopg2cffi'], 'psycopg2 / pg8000 - specific feature') @engines.close_open_connections def test_autocommit_isolation_level(self): diff --git a/test/dialect/postgresql/test_query.py b/test/dialect/postgresql/test_query.py index 26ff5e93b..73319438d 100644 --- a/test/dialect/postgresql/test_query.py +++ b/test/dialect/postgresql/test_query.py @@ -729,6 +729,7 @@ class MatchTest(fixtures.TestBase, AssertsCompiledSQL): @testing.fails_on('postgresql+psycopg2', 'uses pyformat') @testing.fails_on('postgresql+pypostgresql', 'uses pyformat') @testing.fails_on('postgresql+zxjdbc', 'uses qmark') + @testing.fails_on('postgresql+psycopg2cffi', 'uses pyformat') def test_expression_positional(self): self.assert_compile(matchtable.c.title.match('somstr'), 'matchtable.title @@ to_tsquery(%s)') diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index c62ca79a8..5f1d37b24 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -379,10 +379,11 @@ class NumericInterpretationTest(fixtures.TestBase): __backend__ = True def test_numeric_codes(self): - from sqlalchemy.dialects.postgresql import pg8000, psycopg2, base - - for dialect in (pg8000.dialect(), psycopg2.dialect()): + from sqlalchemy.dialects.postgresql import psycopg2cffi, pg8000, \ + psycopg2, base + dialects = pg8000.dialect(), psycopg2.dialect(), psycopg2cffi.dialect() + for dialect in dialects: typ = Numeric().dialect_impl(dialect) for code in base._INT_TYPES + base._FLOAT_TYPES + \ base._DECIMAL_TYPES: diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index 725dcebe0..b5b414af2 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -174,7 +174,7 @@ class ExecuteTest(fixtures.TestBase): @testing.skip_if( lambda: testing.against('mysql+mysqldb'), 'db-api flaky') @testing.fails_on_everything_except( - 'postgresql+psycopg2', + 'postgresql+psycopg2', 'postgresql+psycopg2cffi', 'postgresql+pypostgresql', 'mysql+mysqlconnector', 'mysql+pymysql', 'mysql+cymysql') def test_raw_python(self): diff --git a/test/requirements.py b/test/requirements.py index ffbdfba23..6b8ba504c 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -756,6 +756,10 @@ class DefaultRequirements(SuiteRequirements): "+psycopg2", None, None, "psycopg2 2.4 no longer accepts percent " "sign in bind placeholders"), + ( + "+psycopg2cffi", None, None, + "psycopg2cffi does not accept percent signs in " + "bind placeholders"), ("mysql", None, None, "executemany() doesn't work here") ] ) diff --git a/test/sql/test_types.py b/test/sql/test_types.py index 38b3ced13..5e1542853 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -952,6 +952,7 @@ class UnicodeTest(fixtures.TestBase): expected = (testing.db.name, testing.db.driver) in \ ( ('postgresql', 'psycopg2'), + ('postgresql', 'psycopg2cffi'), ('postgresql', 'pypostgresql'), ('postgresql', 'pg8000'), ('postgresql', 'zxjdbc'), -- cgit v1.2.1 From 0953f2625046b98c7b6fbe157942eddde657e08a Mon Sep 17 00:00:00 2001 From: Shaun Stanworth Date: Sun, 9 Nov 2014 15:18:04 +0000 Subject: 78-char width --- test/dialect/postgresql/test_dialect.py | 3 ++- test/dialect/postgresql/test_types.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/dialect/postgresql/test_dialect.py b/test/dialect/postgresql/test_dialect.py index 2166bca32..9f86aaa7a 100644 --- a/test/dialect/postgresql/test_dialect.py +++ b/test/dialect/postgresql/test_dialect.py @@ -118,7 +118,8 @@ class MiscTest(fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL): eq_(c.connection.connection.encoding, test_encoding) @testing.only_on( - ['postgresql+psycopg2', 'postgresql+pg8000', 'postgresql+psycopg2cffi'], + ['postgresql+psycopg2', 'postgresql+pg8000', + 'postgresql+psycopg2cffi'], 'psycopg2 / pg8000 - specific feature') @engines.close_open_connections def test_autocommit_isolation_level(self): diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index 5f1d37b24..95c034f11 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -382,7 +382,8 @@ class NumericInterpretationTest(fixtures.TestBase): from sqlalchemy.dialects.postgresql import psycopg2cffi, pg8000, \ psycopg2, base - dialects = pg8000.dialect(), psycopg2.dialect(), psycopg2cffi.dialect() + dialects = (pg8000.dialect(), psycopg2.dialect(), + psycopg2cffi.dialect()) for dialect in dialects: typ = Numeric().dialect_impl(dialect) for code in base._INT_TYPES + base._FLOAT_TYPES + \ -- cgit v1.2.1 From 226bd8d7077c3557e5fc3b7ad512c4ad59b471d1 Mon Sep 17 00:00:00 2001 From: Shaun Stanworth Date: Sun, 9 Nov 2014 15:18:16 +0000 Subject: Include psycopg2cffi in dialect docs --- doc/build/dialects/postgresql.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/build/dialects/postgresql.rst b/doc/build/dialects/postgresql.rst index e1a96493e..11bbe2cf9 100644 --- a/doc/build/dialects/postgresql.rst +++ b/doc/build/dialects/postgresql.rst @@ -202,3 +202,8 @@ zxjdbc -------------- .. automodule:: sqlalchemy.dialects.postgresql.zxjdbc + +psycopg2cffi +-------------- + +.. automodule:: sqlalchemy.dialects.postgresql.psycopg2cffi -- cgit v1.2.1 From a826ff366bf979bcbb4f075d3c33540232b91873 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 26 Jan 2015 18:02:46 -0500 Subject: - additional test adjustments for pypy / psycopg2cffi. This consists mainly of adjusting fixtures to ensure connections are closed explicitly. psycopg2cffi also handles unicode bind parameter names differently than psycopg2, and seems to possibly have a little less control over floating point values at least in one test which is marked as a "fail", though will see if it runs differently on linux than osx.. - changelog for psycopg2cffi, fixes #3052 --- doc/build/changelog/changelog_10.rst | 12 ++++++ doc/build/changelog/migration_10.rst | 8 ++++ doc/build/dialects/postgresql.rst | 18 ++++---- lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py | 9 ++-- lib/sqlalchemy/testing/util.py | 4 ++ setup.cfg | 2 +- test/dialect/postgresql/test_query.py | 32 +++++++------- test/dialect/postgresql/test_types.py | 4 +- test/engine/test_execute.py | 50 +++++++++++----------- test/requirements.py | 4 ++ 10 files changed, 90 insertions(+), 53 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 18043c456..2c3e26f2e 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -22,6 +22,18 @@ series as well. For changes that are specific to 1.0 with an emphasis on compatibility concerns, see :doc:`/changelog/migration_10`. + .. change:: + :tags: feature, postgresql, pypy + :tickets: 3052 + :pullreq: bitbucket:34 + + Added support for the psycopg2cffi DBAPI on pypy. Pull request + courtesy shauns. + + .. seealso:: + + :mod:`sqlalchemy.dialects.postgresql.psycopg2cffi` + .. change:: :tags: feature, orm :tickets: 3262 diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index c0369d8b8..23ee6f466 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -1769,6 +1769,14 @@ by Postgresql as of 9.4. SQLAlchemy allows this using :class:`.FunctionFilter` +Support for psycopg2cffi Dialect on Pypy +---------------------------------------- + +Support for the pypy psycopg2cffi dialect is added. + +.. seealso:: + + :mod:`sqlalchemy.dialects.postgresql.psycopg2cffi` Dialect Improvements and Changes - MySQL ============================================= diff --git a/doc/build/dialects/postgresql.rst b/doc/build/dialects/postgresql.rst index 11bbe2cf9..e5d8d51bc 100644 --- a/doc/build/dialects/postgresql.rst +++ b/doc/build/dialects/postgresql.rst @@ -188,22 +188,24 @@ psycopg2 .. automodule:: sqlalchemy.dialects.postgresql.psycopg2 +pg8000 +-------------- + +.. automodule:: sqlalchemy.dialects.postgresql.pg8000 + +psycopg2cffi +-------------- + +.. automodule:: sqlalchemy.dialects.postgresql.psycopg2cffi + py-postgresql -------------------- .. automodule:: sqlalchemy.dialects.postgresql.pypostgresql -pg8000 --------------- - -.. automodule:: sqlalchemy.dialects.postgresql.pg8000 zxjdbc -------------- .. automodule:: sqlalchemy.dialects.postgresql.zxjdbc -psycopg2cffi --------------- - -.. automodule:: sqlalchemy.dialects.postgresql.psycopg2cffi diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py b/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py index 5217c5561..f5c475d90 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py @@ -1,5 +1,5 @@ # testing/engines.py -# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -13,9 +13,11 @@ postgresql+psycopg2cffi://user:password@host:port/dbname\ [?key=value&key=value...] :url: http://pypi.python.org/pypi/psycopg2cffi/ -`psycopg2cffi` is an adaptation of `psycopg2`, using CFFI for the C +``psycopg2cffi`` is an adaptation of ``psycopg2``, using CFFI for the C layer. This makes it suitable for use in e.g. PyPy. Documentation -is as per `psycopg2`. +is as per ``psycopg2``. + +.. versionadded:: 1.0.0 .. seealso:: @@ -27,6 +29,7 @@ from .psycopg2 import PGDialect_psycopg2 class PGDialect_psycopg2cffi(PGDialect_psycopg2): driver = 'psycopg2cffi' + supports_unicode_statements = True @classmethod def dbapi(cls): diff --git a/lib/sqlalchemy/testing/util.py b/lib/sqlalchemy/testing/util.py index eea39b1f7..8230f923a 100644 --- a/lib/sqlalchemy/testing/util.py +++ b/lib/sqlalchemy/testing/util.py @@ -147,6 +147,10 @@ def run_as_contextmanager(ctx, fn, *arg, **kw): simulating the behavior of 'with' to support older Python versions. + This is not necessary anymore as we have placed 2.6 + as minimum Python version, however some tests are still using + this structure. + """ obj = ctx.__enter__() diff --git a/setup.cfg b/setup.cfg index 5eb35469f..dc10877f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ postgresql=postgresql://scott:tiger@127.0.0.1:5432/test pg8000=postgresql+pg8000://scott:tiger@127.0.0.1:5432/test postgres=postgresql://scott:tiger@127.0.0.1:5432/test postgresql_jython=postgresql+zxjdbc://scott:tiger@127.0.0.1:5432/test -postgresql_psycopg2cffi=postgresql+psycopg2cffi://127.0.0.1:5432/test +postgresql_psycopg2cffi=postgresql+psycopg2cffi://scott:tiger@127.0.0.1:5432/test mysql=mysql://scott:tiger@127.0.0.1:3306/test mysqlconnector=mysql+mysqlconnector://scott:tiger@127.0.0.1:3306/test mssql=mssql+pyodbc://scott:tiger@ms_2008 diff --git a/test/dialect/postgresql/test_query.py b/test/dialect/postgresql/test_query.py index 73319438d..27cb958fd 100644 --- a/test/dialect/postgresql/test_query.py +++ b/test/dialect/postgresql/test_query.py @@ -856,21 +856,23 @@ class ExtractTest(fixtures.TablesTest): def utcoffset(self, dt): return datetime.timedelta(hours=4) - conn = testing.db.connect() - - # we aren't resetting this at the moment but we don't have - # any other tests that are TZ specific - conn.execute("SET SESSION TIME ZONE 0") - conn.execute( - cls.tables.t.insert(), - { - 'dtme': datetime.datetime(2012, 5, 10, 12, 15, 25), - 'dt': datetime.date(2012, 5, 10), - 'tm': datetime.time(12, 15, 25), - 'intv': datetime.timedelta(seconds=570), - 'dttz': datetime.datetime(2012, 5, 10, 12, 15, 25, tzinfo=TZ()) - }, - ) + with testing.db.connect() as conn: + + # we aren't resetting this at the moment but we don't have + # any other tests that are TZ specific + conn.execute("SET SESSION TIME ZONE 0") + conn.execute( + cls.tables.t.insert(), + { + 'dtme': datetime.datetime(2012, 5, 10, 12, 15, 25), + 'dt': datetime.date(2012, 5, 10), + 'tm': datetime.time(12, 15, 25), + 'intv': datetime.timedelta(seconds=570), + 'dttz': + datetime.datetime(2012, 5, 10, 12, 15, 25, + tzinfo=TZ()) + }, + ) def _test(self, expr, field="all", overrides=None): t = self.tables.t diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index 95c034f11..1f572c9a1 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -1399,7 +1399,7 @@ class HStoreRoundTripTest(fixtures.TablesTest): use_native_hstore=False)) else: engine = testing.db - engine.connect() + engine.connect().close() return engine def test_reflect(self): @@ -2031,7 +2031,7 @@ class JSONRoundTripTest(fixtures.TablesTest): engine = engines.testing_engine(options=options) else: engine = testing.db - engine.connect() + engine.connect().close() return engine def test_reflect(self): diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index b5b414af2..730ef4446 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -639,21 +639,21 @@ class ConvenienceExecuteTest(fixtures.TablesTest): def test_transaction_connection_ctx_commit(self): fn = self._trans_fn(True) - conn = testing.db.connect() - ctx = conn.begin() - testing.run_as_contextmanager(ctx, fn, 5, value=8) - self._assert_fn(5, value=8) + with testing.db.connect() as conn: + ctx = conn.begin() + testing.run_as_contextmanager(ctx, fn, 5, value=8) + self._assert_fn(5, value=8) def test_transaction_connection_ctx_rollback(self): fn = self._trans_rollback_fn(True) - conn = testing.db.connect() - ctx = conn.begin() - assert_raises_message( - Exception, - "breakage", - testing.run_as_contextmanager, ctx, fn, 5, value=8 - ) - self._assert_no_data() + with testing.db.connect() as conn: + ctx = conn.begin() + assert_raises_message( + Exception, + "breakage", + testing.run_as_contextmanager, ctx, fn, 5, value=8 + ) + self._assert_no_data() def test_connection_as_ctx(self): fn = self._trans_fn() @@ -666,10 +666,12 @@ class ConvenienceExecuteTest(fixtures.TablesTest): def test_connect_as_ctx_noautocommit(self): fn = self._trans_fn() self._assert_no_data() - ctx = testing.db.connect().execution_options(autocommit=False) - testing.run_as_contextmanager(ctx, fn, 5, value=8) - # autocommit is off - self._assert_no_data() + + with testing.db.connect() as conn: + ctx = conn.execution_options(autocommit=False) + testing.run_as_contextmanager(ctx, fn, 5, value=8) + # autocommit is off + self._assert_no_data() def test_transaction_engine_fn_commit(self): fn = self._trans_fn() @@ -687,17 +689,17 @@ class ConvenienceExecuteTest(fixtures.TablesTest): def test_transaction_connection_fn_commit(self): fn = self._trans_fn() - conn = testing.db.connect() - conn.transaction(fn, 5, value=8) - self._assert_fn(5, value=8) + with testing.db.connect() as conn: + conn.transaction(fn, 5, value=8) + self._assert_fn(5, value=8) def test_transaction_connection_fn_rollback(self): fn = self._trans_rollback_fn() - conn = testing.db.connect() - assert_raises( - Exception, - conn.transaction, fn, 5, value=8 - ) + with testing.db.connect() as conn: + assert_raises( + Exception, + conn.transaction, fn, 5, value=8 + ) self._assert_no_data() diff --git a/test/requirements.py b/test/requirements.py index 6b8ba504c..89fc108b9 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -656,6 +656,10 @@ class DefaultRequirements(SuiteRequirements): 'postgresql+pg8000', None, None, 'postgresql+pg8000 has FP inaccuracy even with ' 'only four decimal places '), + ( + 'postgresql+psycopg2cffi', None, None, + 'postgresql+psycopg2cffi has FP inaccuracy even with ' + 'only four decimal places '), ]) @property -- cgit v1.2.1 From 62f87749067684696dca32cacac17f7d33066d8b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 26 Jan 2015 22:45:12 -0500 Subject: - fix this test for py3k --- test/sql/test_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sql/test_defaults.py b/test/sql/test_defaults.py index ce11a2603..48505dd8c 100644 --- a/test/sql/test_defaults.py +++ b/test/sql/test_defaults.py @@ -379,7 +379,7 @@ class DefaultTest(fixtures.TestBase): assert_raises_message( sa.exc.ArgumentError, "SQL expression object or string expected, got object of type " - " instead", + "<.* 'list'> instead", t.select, [const] ) assert_raises_message( -- cgit v1.2.1 From 8aaa8dd6bdfb85fa481efa3115b9080d935d344c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 27 Jan 2015 00:34:10 -0500 Subject: - fix link to non_primary flag - rewrite the multiple mappers section --- doc/build/glossary.rst | 1 + doc/build/orm/nonstandard_mappings.rst | 64 +++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/doc/build/glossary.rst b/doc/build/glossary.rst index 2e5dc67f3..c0ecee84b 100644 --- a/doc/build/glossary.rst +++ b/doc/build/glossary.rst @@ -99,6 +99,7 @@ Glossary instrumentation instrumented + instrumenting Instrumentation refers to the process of augmenting the functionality and attribute set of a particular class. Ideally, the behavior of the class should remain close to a regular diff --git a/doc/build/orm/nonstandard_mappings.rst b/doc/build/orm/nonstandard_mappings.rst index b3733a1b9..4645a8029 100644 --- a/doc/build/orm/nonstandard_mappings.rst +++ b/doc/build/orm/nonstandard_mappings.rst @@ -123,30 +123,46 @@ key. Multiple Mappers for One Class ============================== -In modern SQLAlchemy, a particular class is only mapped by one :func:`.mapper` -at a time. The rationale here is that the :func:`.mapper` modifies the class itself, not only -persisting it towards a particular :class:`.Table`, but also *instrumenting* +In modern SQLAlchemy, a particular class is mapped by only one so-called +**primary** mapper at a time. This mapper is involved in three main +areas of functionality: querying, persistence, and instrumentation of the +mapped class. The rationale of the primary mapper relates to the fact +that the :func:`.mapper` modifies the class itself, not only +persisting it towards a particular :class:`.Table`, but also :term:`instrumenting` attributes upon the class which are structured specifically according to the -table metadata. - -One potential use case for another mapper to exist at the same time is if we -wanted to load instances of our class not just from the immediate :class:`.Table` -to which it is mapped, but from another selectable that is a derivation of that -:class:`.Table`. To create a second mapper that only handles querying -when used explicitly, we can use the :paramref:`.mapper.non_primary` argument. -In practice, this approach is usually not needed, as we -can do this sort of thing at query time using methods such as -:meth:`.Query.select_from`, however it is useful in the rare case that we -wish to build a :func:`.relationship` to such a mapper. An example of this is -at :ref:`relationship_non_primary_mapper`. - -Another potential use is if we genuinely want instances of our class to -be persisted into different tables at different times; certain kinds of -data sharding configurations may persist a particular class into tables -that are identical in structure except for their name. For this kind of -pattern, Python offers a better approach than the complexity of mapping -the same class multiple times, which is to instead create new mapped classes -for each target table. SQLAlchemy refers to this as the "entity name" -pattern, which is described as a recipe at `Entity Name +table metadata. It's not possible for more than one mapper +to be associated with a class in equal measure, since only one mapper can +actually instrument the class. + +However, there is a class of mapper known as the **non primary** mapper +with allows additional mappers to be associated with a class, but with +a limited scope of use. This scope typically applies to +being able to load rows from an alternate table or selectable unit, but +still producing classes which are ultimately persisted using the primary +mapping. The non-primary mapper is created using the classical style +of mapping against a class that is already mapped with a primary mapper, +and involves the use of the :paramref:`~sqlalchemy.orm.mapper.non_primary` +flag. + +The non primary mapper is of very limited use in modern SQLAlchemy, as the +task of being able to load classes from subqueries or other compound statements +can be now accomplished using the :class:`.Query` object directly. + +There is really only one use case for the non-primary mapper, which is that +we wish to build a :func:`.relationship` to such a mapper; this is useful +in the rare and advanced case that our relationship is attempting to join two +classes together using many tables and/or joins in between. An example of this +pattern is at :ref:`relationship_non_primary_mapper`. + +As far as the use case of a class that can actually be fully persisted +to different tables under different scenarios, very early versions of +SQLAlchemy offered a feature for this adapted from Hibernate, known +as the "entity name" feature. However, this use case became infeasable +within SQLAlchemy once the mapped class itself became the source of SQL +expression construction; that is, the class' attributes themselves link +directly to mapped table columns. The feature was removed and replaced +with a simple recipe-oriented approach to accomplishing this task +without any ambiguity of instrumentation - to create new subclasses, each +mapped individually. This pattern is now available as a recipe at `Entity Name `_. -- cgit v1.2.1