- defaultdict benchmarks faster than a namedtuple; OK

- inline the column-based expiration operations as well

4 files changed, 109 insertions, 108 deletions

diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py
index bcb783480..d3d170d94 100644
--- a/lib/sqlalchemy/orm/loading.py
+++ b/lib/sqlalchemy/orm/loading.py
@@ -213,9 +213,6 @@ def load_on_ident(query, key,
     except orm_exc.NoResultFound:
         return None
 
-_populator_struct = collections.namedtuple(
-    'populators', ['new', 'existing', 'eager', 'delayed'])
-
 
 def instance_processor(mapper, context, result, path, adapter,
                        polymorphic_from=None,
@@ -258,7 +255,7 @@ def instance_processor(mapper, context, result, path, adapter,
 
     identity_class = mapper._identity_class
 
-    populators = _populator_struct([], [], [], [])
+    populators = collections.defaultdict(list)
 
     props = mapper._props.values()
     if only_load_props is not None:
@@ -268,16 +265,14 @@ def instance_processor(mapper, context, result, path, adapter,
         prop.create_row_processor(
             context, path, mapper, result, adapter, populators)
 
-    if populators.delayed:
-        populators.new.extend(populators.delayed)
-
-    (new_populators, existing_populators,
-        eager_populators) = (
-        populators.new, populators.existing, populators.eager)
+    quick_populators = populators.get('quick', ())
+    expire_populators = populators.get('expire', ())
+    new_populators = populators.get('new', []) + populators.get('delayed', [])
+    existing_populators = populators.get('existing', ())
+    eager_populators = populators.get('eager', ())
 
     load_path = context.query._current_path + path \
-        if context.query._current_path.path \
-        else path
+        if context.query._current_path.path else path
 
     session_identity_map = context.session.identity_map
 
@@ -380,69 +375,96 @@ def instance_processor(mapper, context, result, path, adapter,
             session_identity_map._add_unpresent(state, identitykey)
 
         if currentload or populate_existing:
-            # state is being fully loaded, so populate.
-            # add to the "context.progress" collection.
+            # full population routines.  Objects here are either
+            # just created, or we are doing a populate_existing
             if isnew:
+                # first time we are seeing a row with this identity.
                 state.runid = runid
                 if propagate_options:
                     state.load_options = propagate_options
                 if state.load_options:
                     state.load_path = load_path
+
+                for key, getter in quick_populators:
+                    dict_[key] = getter(row)
+                if populate_existing:
+                    for key, set_callable in expire_populators:
+                        dict_.pop(key, None)
+                        if set_callable:
+                            state.callables[key] = state
+                else:
+                    for key, set_callable in expire_populators:
+                        if set_callable:
+                            state.callables[key] = state
                 for key, populator in new_populators:
                     populator(state, dict_, row)
+
+                if loaded_instance and load_evt:
+                    state.manager.dispatch.load(state, context)
+                elif isnew and refresh_evt:
+                    state.manager.dispatch.refresh(
+                        state, context, only_load_props)
+
+                if populate_existing or state.modified:
+                    if refresh_state and only_load_props:
+                        state._commit(dict_, only_load_props)
+                    else:
+                        state._commit_all(dict_, session_identity_map)
             else:
+                # have already seen rows with this identity.
                 for key, populator in existing_populators:
                     populator(state, dict_, row)
-
-            if loaded_instance and load_evt:
-                state.manager.dispatch.load(state, context)
-            elif isnew and refresh_evt:
-                state.manager.dispatch.refresh(
-                    state, context, only_load_props)
-
-            if populate_existing or state.modified:
-                if refresh_state and only_load_props:
-                    state._commit(dict_, only_load_props)
-                else:
-                    state._commit_all(dict_, session_identity_map)
-
-        elif state in context.partials or state.unloaded or eager_populators:
-            # state is having a partial set of its attributes
-            # refreshed.  Populate those attributes,
-            # and add to the "context.partials" collection.
+        else:
+            # partial population routines, for objects that were already
+            # in the Session, but a row matches them; apply eager loaders
+            # on existing objects, etc.
             unloaded = state.unloaded
-
-            if state in context.partials:
-                isnew = False
-                to_load = context.partials[state]
-                for key, populator in existing_populators:
-                    if key not in to_load:
-                        continue
-                    populator(state, dict_, row)
-            else:
-                isnew = True
-                to_load = unloaded
-                context.partials[state] = to_load
-
-                if context.propagate_options:
-                    state.load_options = context.propagate_options
-                if state.load_options:
-                    state.load_path = load_path
-
-                for key, populator in new_populators:
-                    if key not in to_load:
-                        continue
-                    populator(state, dict_, row)
-
-            for key, pop in eager_populators:
-                if key not in unloaded:
-                    pop(state, dict_, row)
-
-            if isnew and refresh_evt:
-                state.manager.dispatch.refresh(state, context, to_load)
-
-            if isnew:
-                state._commit(dict_, to_load)
+            isnew = state not in context.partials
+
+            if not isnew or unloaded or eager_populators:
+                # state is having a partial set of its attributes
+                # refreshed.  Populate those attributes,
+                # and add to the "context.partials" collection.
+
+                if not isnew:
+                    to_load = context.partials[state]
+                    for key, populator in existing_populators:
+                        if key not in to_load:
+                            continue
+                        populator(state, dict_, row)
+                else:
+                    to_load = unloaded
+                    context.partials[state] = to_load
+
+                    if context.propagate_options:
+                        state.load_options = context.propagate_options
+                    if state.load_options:
+                        state.load_path = load_path
+
+                    for key, getter in quick_populators:
+                        if key not in to_load:
+                            continue
+                        dict_[key] = getter(row)
+                    for key, set_callable in expire_populators:
+                        if key not in to_load:
+                            continue
+                        dict_.pop(key, None)
+                        if set_callable:
+                            state.callables[key] = state
+                    for key, populator in new_populators:
+                        if key not in to_load:
+                            continue
+                        populator(state, dict_, row)
+
+                for key, pop in eager_populators:
+                    if key not in unloaded:
+                        pop(state, dict_, row)
+
+                if isnew:
+                    if refresh_evt:
+                        state.manager.dispatch.refresh(state, context, to_load)
+
+                    state._commit(dict_, to_load)
 
         return instance
     return _instance
diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py
index 17872a9b4..3c12fda1a 100644
--- a/lib/sqlalchemy/orm/state.py
+++ b/lib/sqlalchemy/orm/state.py
@@ -334,20 +334,6 @@ class InstanceState(interfaces.InspectionAttr):
             self.manager[key].impl._invalidate_collection(old)
         self.callables.pop(key, None)
 
-    def _expire_attribute_pre_commit(self, dict_, key):
-        """a fast expire that can be called by column loaders during a load.
-
-        The additional bookkeeping is finished up in commit_all().
-
-        Should only be called for scalar attributes.
-
-        This method is actually called a lot with joined-table
-        loading, when the second table isn't present in the result.
-
-        """
-        dict_.pop(key, None)
-        self.callables[key] = self
-
     @classmethod
     def _row_processor(cls, manager, fn, key):
         impl = manager[key].impl
diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py
index aa52d91da..e31b3ae6d 100644
--- a/lib/sqlalchemy/orm/strategies.py
+++ b/lib/sqlalchemy/orm/strategies.py
@@ -158,7 +158,6 @@ class ColumnLoader(LoaderStrategy):
     def create_row_processor(
             self, context, path,
             loadopt, mapper, result, adapter, populators):
-        key = self.key
         # look through list of columns represented here
         # to see which, if any, is present in the row.
         for col in self.columns:
@@ -166,14 +165,10 @@ class ColumnLoader(LoaderStrategy):
                 col = adapter.columns[col]
             getter = result._getter(col)
             if getter:
-                def fetch_col(state, dict_, row):
-                    dict_[key] = getter(row)
-                populators.new.append((self.key, fetch_col))
+                populators["quick"].append((self.key, getter))
                 break
         else:
-            def expire_for_non_present_col(state, dict_, row):
-                state._expire_attribute_pre_commit(dict_, key)
-            populators.new.append((self.key, expire_for_non_present_col))
+            populators["expire"].append((self.key, True))
 
 
 @log.class_logger
@@ -196,8 +191,6 @@ class DeferredColumnLoader(LoaderStrategy):
         if adapter:
             col = adapter.columns[col]
 
-        key = self.key
-
         # TODO: put a result-level contains here
         getter = result._getter(col)
         if getter:
@@ -209,14 +202,10 @@ class DeferredColumnLoader(LoaderStrategy):
         elif not self.is_class_level:
             set_deferred_for_local_state = InstanceState._row_processor(
                 mapper.class_manager,
-                LoadDeferredColumns(key), key)
-            populators.new.append((self.key, set_deferred_for_local_state))
+                LoadDeferredColumns(self.key), self.key)
+            populators["new"].append((self.key, set_deferred_for_local_state))
         else:
-            def reset_col_for_deferred(state, dict_, row):
-                # reset state on the key so that deferred callables
-                # fire off on next access.
-                state._reset(dict_, key)
-            populators.new.append((self.key, reset_col_for_deferred))
+            populators["expire"].append((self.key, False))
 
     def init_class_attribute(self, mapper):
         self.is_class_level = True
@@ -342,7 +331,7 @@ class NoLoader(AbstractRelationshipLoader):
             result, adapter, populators):
         def invoke_no_load(state, dict_, row):
             state._initialize(self.key)
-        populators.new.append((self.key, invoke_no_load))
+        populators["new"].append((self.key, invoke_no_load))
 
 
 @log.class_logger
@@ -639,7 +628,7 @@ class LazyLoader(AbstractRelationshipLoader):
                 mapper.class_manager,
                 LoadLazyAttribute(key), key)
 
-            populators.new.append((self.key, set_lazy_callable))
+            populators["new"].append((self.key, set_lazy_callable))
         elif context.populate_existing or mapper.always_refresh:
             def reset_for_lazy_callable(state, dict_, row):
                 # we are the primary manager for this attribute on
@@ -652,7 +641,7 @@ class LazyLoader(AbstractRelationshipLoader):
                 # any existing state.
                 state._reset(dict_, key)
 
-            populators.new.append((self.key, reset_for_lazy_callable))
+            populators["new"].append((self.key, reset_for_lazy_callable))
 
 
 class LoadLazyAttribute(object):
@@ -689,7 +678,7 @@ class ImmediateLoader(AbstractRelationshipLoader):
         def load_immediate(state, dict_, row):
             state.get_impl(self.key).get(state, dict_)
 
-        populators.delayed.append((self.key, load_immediate))
+        populators["delayed"].append((self.key, load_immediate))
 
 
 @log.class_logger
@@ -1046,9 +1035,9 @@ class SubqueryLoader(AbstractRelationshipLoader):
             state.get_impl(self.key).\
                 set_committed_value(state, dict_, collection)
 
-        populators.new.append((self.key, load_collection_from_subq))
+        populators["new"].append((self.key, load_collection_from_subq))
         if context.invoke_all_eagers:
-            populators.eager.append((self.key, collections.loader))
+            populators["eager"].append((self.key, collections.loader))
 
     def _create_scalar_loader(
             self, context, collections, local_cols, populators):
@@ -1067,9 +1056,9 @@ class SubqueryLoader(AbstractRelationshipLoader):
             state.get_impl(self.key).\
                 set_committed_value(state, dict_, scalar)
 
-        populators.new.append((self.key, load_scalar_from_subq))
+        populators["new"].append((self.key, load_scalar_from_subq))
         if context.invoke_all_eagers:
-            populators.eager.append((self.key, collections.loader))
+            populators["eager"].append((self.key, collections.loader))
 
 
 @log.class_logger
@@ -1491,11 +1480,11 @@ class JoinedLoader(AbstractRelationshipLoader):
         def load_collection_from_joined_exec(state, dict_, row):
             _instance(row)
 
-        populators.new.append((self.key, load_collection_from_joined_new_row))
-        populators.existing.append(
+        populators["new"].append((self.key, load_collection_from_joined_new_row))
+        populators["existing"].append(
             (self.key, load_collection_from_joined_existing_row))
         if context.invoke_all_eagers:
-            populators.eager.append(
+            populators["eager"].append(
                 (self.key, load_collection_from_joined_exec))
 
     def _create_scalar_loader(self, context, key, _instance, populators):
@@ -1519,11 +1508,11 @@ class JoinedLoader(AbstractRelationshipLoader):
         def load_scalar_from_joined_exec(state, dict_, row):
             _instance(row)
 
-        populators.new.append((self.key, load_scalar_from_joined_new_row))
-        populators.existing.append(
+        populators["new"].append((self.key, load_scalar_from_joined_new_row))
+        populators["existing"].append(
             (self.key, load_scalar_from_joined_existing_row))
         if context.invoke_all_eagers:
-            populators.eager.append((self.key, load_scalar_from_joined_exec))
+            populators["eager"].append((self.key, load_scalar_from_joined_exec))
 
 
 def single_parent_validator(desc, prop):
diff --git a/test/orm/test_attributes.py b/test/orm/test_attributes.py
index 59b0078ea..36ad64506 100644
--- a/test/orm/test_attributes.py
+++ b/test/orm/test_attributes.py
@@ -1814,7 +1814,11 @@ class HistoryTest(fixtures.TestBase):
         self._commit_someattr(f)
 
         state = attributes.instance_state(f)
-        state._expire_attribute_pre_commit(state.dict, 'someattr')
+        # do the same thing that
+        # populators.expire.append((self.key, True))
+        # does in loading.py
+        state.dict.pop('someattr', None)
+        state.callables['someattr'] = state
 
         def scalar_loader(state, toload):
             state.dict['someattr'] = 'one'