From 46f34449f389a8aa994485f36e99f95275a19170 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 16 Apr 2021 23:58:48 -0400 Subject: Uniquify FROMs when traversing through select Fixed a critical performance issue where the traversal of a :func:`_sql.select` construct would traverse a repetitive product of the represented FROM clauses as they were each referred towards by columns in the columns clause; for a series of nested subqueries with lots of columns this could cause a large delay and significant memory growth. This traversal is used by a wide variety of SQL and ORM functions, including by the ORM :class:`_orm.Session` when it's configured to have "table-per-bind", which while this is not a common use case, it seems to be what Flask-SQLAlchemy is hardcoded as using, so the issue impacts Flask-SQLAlchemy users. The traversal has been repaired to uniqify on FROM clauses which was effectively what would happen implicitly with the pre-1.4 architecture. Fixes: #6304 Change-Id: I43497e943db4065deab0bfc830fbb68c17b80a53 --- lib/sqlalchemy/sql/selectable.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'lib/sqlalchemy/sql') diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py index 2a221cdf0..3a90f77fb 100644 --- a/lib/sqlalchemy/sql/selectable.py +++ b/lib/sqlalchemy/sql/selectable.py @@ -4428,15 +4428,24 @@ class _SelectFromElements(object): # note this does not include elements # in _setup_joins or _legacy_setup_joins - return itertools.chain( - itertools.chain.from_iterable( - [element._from_objects for element in self._raw_columns] - ), - itertools.chain.from_iterable( - [element._from_objects for element in self._where_criteria] - ), - self._from_obj, - ) + seen = set() + for element in self._raw_columns: + for fr in element._from_objects: + if fr in seen: + continue + seen.add(fr) + yield fr + for element in self._where_criteria: + for fr in element._from_objects: + if fr in seen: + continue + seen.add(fr) + yield fr + for element in self._from_obj: + if element in seen: + continue + seen.add(element) + yield element class Select( -- cgit v1.2.1