immediateload lazy relationships named in refresh.attribute_names

The :meth:`_orm.Session.refresh` method will now immediately load a
relationship-bound attribute that is explicitly named within the
:paramref:`_orm.Session.refresh.attribute_names` collection even if it is
currently linked to the "select" loader, which normally is a "lazy" loader
that does not fire off during a refresh. The "lazy loader" strategy will
now detect that the operation is specifically a user-initiated
:meth:`_orm.Session.refresh` operation which named this attribute
explicitly, and will then call upon the "immediateload" strategy to
actually emit SQL to load the attribute. This should be helpful in
particular for some asyncio situations where the loading of an unloaded
lazy-loaded attribute must be forced, without using the actual lazy-loading
attribute pattern not supported in asyncio.

Fixes: #9298
Change-Id: I9b50f339bdf06cdb2ec98f8e5efca2b690895dd7

9 files changed, 152 insertions, 32 deletions

diff --git a/doc/build/changelog/unreleased_20/9298.rst b/doc/build/changelog/unreleased_20/9298.rst
new file mode 100644
index 000000000..f9150eb3b
--- /dev/null
+++ b/doc/build/changelog/unreleased_20/9298.rst
@@ -0,0 +1,17 @@
+.. change::
+    :tags: usecase, orm
+    :tickets: 9298
+
+    The :meth:`_orm.Session.refresh` method will now immediately load a
+    relationship-bound attribute that is explicitly named within the
+    :paramref:`_orm.Session.refresh.attribute_names` collection even if it is
+    currently linked to the "select" loader, which normally is a "lazy" loader
+    that does not fire off during a refresh. The "lazy loader" strategy will
+    now detect that the operation is specifically a user-initiated
+    :meth:`_orm.Session.refresh` operation which named this attribute
+    explicitly, and will then call upon the "immediateload" strategy to
+    actually emit SQL to load the attribute. This should be helpful in
+    particular for some asyncio situations where the loading of an unloaded
+    lazy-loaded attribute must be forced, without using the actual lazy-loading
+    attribute pattern not supported in asyncio.
+
diff --git a/doc/build/orm/extensions/asyncio.rst b/doc/build/orm/extensions/asyncio.rst
index 322c5081a..59989ad4e 100644
--- a/doc/build/orm/extensions/asyncio.rst
+++ b/doc/build/orm/extensions/asyncio.rst
@@ -337,6 +337,31 @@ Other guidelines include:
   :paramref:`_orm.Session.expire_on_commit`
   should normally be set to ``False`` when using asyncio.
 
+* A lazy-loaded relationship **can be loaded explicitly under asyncio** using
+  :meth:`_asyncio.AsyncSession.refresh`, **if** the desired attribute name
+  is passed explicitly to
+  :paramref:`_orm.Session.refresh.attribute_names`, e.g.::
+
+    # assume a_obj is an A that has lazy loaded A.bs collection
+    a_obj = await async_session.get(A, [1])
+
+    # force the collection to load by naming it in attribute_names
+    await async_session.refresh(a_obj, ["bs"])
+
+    # collection is present
+    print(f"bs collection: {a_obj.bs}")
+
+  It's of course preferable to use eager loading up front in order to have
+  collections already set up without the need to lazy-load.
+
+  .. versionadded:: 2.0.4 Added support for
+     :meth:`_asyncio.AsyncSession.refresh` and the underlying
+     :meth:`_orm.Session.refresh` method to force lazy-loaded relationships
+     to load, if they are named explicitly in the
+     :paramref:`_orm.Session.refresh.attribute_names` parameter.
+     In previous versions, the relationship would be silently skipped even
+     if named in the parameter.
+
 * Avoid using the ``all`` cascade option documented at :ref:`unitofwork_cascades`
   in favor of listing out the desired cascade features explicitly.   The
   ``all`` cascade option implies among others the :ref:`cascade_refresh_expire`
diff --git a/lib/sqlalchemy/orm/context.py b/lib/sqlalchemy/orm/context.py
index e6f14daad..2b45b5adc 100644
--- a/lib/sqlalchemy/orm/context.py
+++ b/lib/sqlalchemy/orm/context.py
@@ -141,6 +141,7 @@ class QueryContext:
         _lazy_loaded_from = None
         _legacy_uniquing = False
         _sa_top_level_orm_context = None
+        _is_user_refresh = False
 
     def __init__(
         self,
diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py
index ff52154b0..54b96c215 100644
--- a/lib/sqlalchemy/orm/loading.py
+++ b/lib/sqlalchemy/orm/loading.py
@@ -469,6 +469,7 @@ def load_on_ident(
     bind_arguments: Mapping[str, Any] = util.EMPTY_DICT,
     execution_options: _ExecuteOptions = util.EMPTY_DICT,
     require_pk_cols: bool = False,
+    is_user_refresh: bool = False,
 ):
     """Load the given identity key from the database."""
     if key is not None:
@@ -490,6 +491,7 @@ def load_on_ident(
         bind_arguments=bind_arguments,
         execution_options=execution_options,
         require_pk_cols=require_pk_cols,
+        is_user_refresh=is_user_refresh,
     )
 
 
@@ -507,6 +509,7 @@ def load_on_pk_identity(
     bind_arguments: Mapping[str, Any] = util.EMPTY_DICT,
     execution_options: _ExecuteOptions = util.EMPTY_DICT,
     require_pk_cols: bool = False,
+    is_user_refresh: bool = False,
 ):
 
     """Load the given primary key identity from the database."""
@@ -651,6 +654,7 @@ def load_on_pk_identity(
         only_load_props=only_load_props,
         refresh_state=refresh_state,
         identity_token=identity_token,
+        is_user_refresh=is_user_refresh,
     )
 
     q._compile_options = new_compile_options
@@ -687,6 +691,7 @@ def _set_get_options(
     only_load_props=None,
     refresh_state=None,
     identity_token=None,
+    is_user_refresh=None,
 ):
 
     compile_options = {}
@@ -703,6 +708,8 @@ def _set_get_options(
     if identity_token:
         load_options["_identity_token"] = identity_token
 
+    if is_user_refresh:
+        load_options["_is_user_refresh"] = is_user_refresh
     if load_options:
         load_opt += load_options
     if compile_options:
diff --git a/lib/sqlalchemy/orm/scoping.py b/lib/sqlalchemy/orm/scoping.py
index aafe03673..b46d26d0b 100644
--- a/lib/sqlalchemy/orm/scoping.py
+++ b/lib/sqlalchemy/orm/scoping.py
@@ -1598,12 +1598,24 @@ class scoped_session(Generic[_S]):
         :func:`_orm.relationship` oriented attributes will also be immediately
         loaded if they were already eagerly loaded on the object, using the
         same eager loading strategy that they were loaded with originally.
-        Unloaded relationship attributes will remain unloaded, as will
-        relationship attributes that were originally lazy loaded.
 
         .. versionadded:: 1.4 - the :meth:`_orm.Session.refresh` method
            can also refresh eagerly loaded attributes.
 
+        :func:`_orm.relationship` oriented attributes that would normally
+        load using the ``select`` (or "lazy") loader strategy will also
+        load **if they are named explicitly in the attribute_names
+        collection**, emitting a SELECT statement for the attribute using the
+        ``immediate`` loader strategy.  If lazy-loaded relationships are not
+        named in :paramref:`_orm.Session.refresh.attribute_names`, then
+        they remain as "lazy loaded" attributes and are not implicitly
+        refreshed.
+
+        .. versionchanged:: 2.0.4  The :meth:`_orm.Session.refresh` method
+           will now refresh lazy-loaded :func:`_orm.relationship` oriented
+           attributes for those which are named explicitly in the
+           :paramref:`_orm.Session.refresh.attribute_names` collection.
+
         .. tip::
 
             While the :meth:`_orm.Session.refresh` method is capable of
diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py
index 6b186d838..1a6b050dc 100644
--- a/lib/sqlalchemy/orm/session.py
+++ b/lib/sqlalchemy/orm/session.py
@@ -2919,12 +2919,24 @@ class Session(_SessionClassMethods, EventTarget):
         :func:`_orm.relationship` oriented attributes will also be immediately
         loaded if they were already eagerly loaded on the object, using the
         same eager loading strategy that they were loaded with originally.
-        Unloaded relationship attributes will remain unloaded, as will
-        relationship attributes that were originally lazy loaded.
 
         .. versionadded:: 1.4 - the :meth:`_orm.Session.refresh` method
            can also refresh eagerly loaded attributes.
 
+        :func:`_orm.relationship` oriented attributes that would normally
+        load using the ``select`` (or "lazy") loader strategy will also
+        load **if they are named explicitly in the attribute_names
+        collection**, emitting a SELECT statement for the attribute using the
+        ``immediate`` loader strategy.  If lazy-loaded relationships are not
+        named in :paramref:`_orm.Session.refresh.attribute_names`, then
+        they remain as "lazy loaded" attributes and are not implicitly
+        refreshed.
+
+        .. versionchanged:: 2.0.4  The :meth:`_orm.Session.refresh` method
+           will now refresh lazy-loaded :func:`_orm.relationship` oriented
+           attributes for those which are named explicitly in the
+           :paramref:`_orm.Session.refresh.attribute_names` collection.
+
         .. tip::
 
             While the :meth:`_orm.Session.refresh` method is capable of
@@ -3004,6 +3016,7 @@ class Session(_SessionClassMethods, EventTarget):
                 # above, however removes the additional unnecessary
                 # call to _autoflush()
                 no_autoflush=True,
+                is_user_refresh=True,
             )
             is None
         ):
diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py
index af63b9f6e..5581e5c7f 100644
--- a/lib/sqlalchemy/orm/strategies.py
+++ b/lib/sqlalchemy/orm/strategies.py
@@ -589,6 +589,30 @@ class AbstractRelationshipLoader(LoaderStrategy):
         self.target = self.parent_property.target
         self.uselist = self.parent_property.uselist
 
+    def _immediateload_create_row_processor(
+        self,
+        context,
+        query_entity,
+        path,
+        loadopt,
+        mapper,
+        result,
+        adapter,
+        populators,
+    ):
+        return self.parent_property._get_strategy(
+            (("lazy", "immediate"),)
+        ).create_row_processor(
+            context,
+            query_entity,
+            path,
+            loadopt,
+            mapper,
+            result,
+            adapter,
+            populators,
+        )
+
 
 @log.class_logger
 @relationships.RelationshipProperty.strategy_for(do_nothing=True)
@@ -1143,6 +1167,23 @@ class LazyLoader(
     ):
         key = self.key
 
+        if (
+            context.load_options._is_user_refresh
+            and context.query._compile_options._only_load_props
+            and self.key in context.query._compile_options._only_load_props
+        ):
+
+            return self._immediateload_create_row_processor(
+                context,
+                query_entity,
+                path,
+                loadopt,
+                mapper,
+                result,
+                adapter,
+                populators,
+            )
+
         if not self.is_class_level or (loadopt and loadopt._extra_criteria):
             # we are not the primary manager for this attribute
             # on this class - set up a
@@ -1312,30 +1353,6 @@ class PostLoader(AbstractRelationshipLoader):
 
         return effective_path, True, execution_options, recursion_depth
 
-    def _immediateload_create_row_processor(
-        self,
-        context,
-        query_entity,
-        path,
-        loadopt,
-        mapper,
-        result,
-        adapter,
-        populators,
-    ):
-        return self.parent_property._get_strategy(
-            (("lazy", "immediate"),)
-        ).create_row_processor(
-            context,
-            query_entity,
-            path,
-            loadopt,
-            mapper,
-            result,
-            adapter,
-            populators,
-        )
-
 
 @relationships.RelationshipProperty.strategy_for(lazy="immediate")
 class ImmediateLoader(PostLoader):
diff --git a/test/ext/asyncio/test_session_py3k.py b/test/ext/asyncio/test_session_py3k.py
index b34578dcc..36135a43d 100644
--- a/test/ext/asyncio/test_session_py3k.py
+++ b/test/ext/asyncio/test_session_py3k.py
@@ -164,6 +164,21 @@ class AsyncSessionQueryTest(AsyncFixture):
         is_(u3, None)
 
     @async_test
+    async def test_force_a_lazyload(self, async_session):
+        """test for #9298"""
+
+        User = self.classes.User
+
+        stmt = select(User).order_by(User.id)
+
+        result = (await async_session.scalars(stmt)).all()
+
+        for user_obj in result:
+            await async_session.refresh(user_obj, ["addresses"])
+
+        eq_(result, self.static.user_address_result)
+
+    @async_test
     async def test_get_loader_options(self, async_session):
         User = self.classes.User
 
diff --git a/test/orm/test_expire.py b/test/orm/test_expire.py
index 6138f4b5d..f98cae922 100644
--- a/test/orm/test_expire.py
+++ b/test/orm/test_expire.py
@@ -960,7 +960,12 @@ class ExpireTest(_fixtures.FixtureTest):
         self.assert_sql_count(testing.db, go, 1)
 
     @testing.combinations(
-        "selectin", "joined", "subquery", "immediate", argnames="lazy"
+        "selectin",
+        "joined",
+        "subquery",
+        "immediate",
+        "select",
+        argnames="lazy",
     )
     @testing.variation(
         "as_option",
@@ -983,7 +988,8 @@ class ExpireTest(_fixtures.FixtureTest):
     def test_load_only_relationships(
         self, lazy, expire_first, include_column, as_option, autoflush
     ):
-        """test #8703, #8997 as well as a regression for #8996"""
+        """test #8703, #8997, a regression for #8996, and new feature
+        for #9298."""
 
         users, Address, addresses, User = (
             self.tables.users,
@@ -1025,6 +1031,7 @@ class ExpireTest(_fixtures.FixtureTest):
                 "selectin": selectinload,
                 "subquery": subqueryload,
                 "immediate": immediateload,
+                "select": lazyload,
             }[lazy]
 
         u = sess.get(
@@ -1088,7 +1095,9 @@ class ExpireTest(_fixtures.FixtureTest):
                 sess.refresh(u, ["addresses"])
                 id_was_refreshed = False
 
-        expected_count = 2 if lazy != "joined" else 1
+        expect_addresses = lazy != "select" or not include_column.no_attrs
+
+        expected_count = 2 if (lazy != "joined" and expect_addresses) else 1
         if (
             autoflush
             and expire_first.not_pk_plus_pending
@@ -1106,7 +1115,11 @@ class ExpireTest(_fixtures.FixtureTest):
         else:
             assert "name" in u.__dict__
 
-        assert "addresses" in u.__dict__
+        if expect_addresses:
+            assert "addresses" in u.__dict__
+        else:
+            assert "addresses" not in u.__dict__
+
         u.addresses
         assert "addresses" in u.__dict__
         if include_column: