Merge "Fix MSSQL / Oracle limit/offset regressions"

4 files changed, 228 insertions, 102 deletions

diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py
index 1be3965b8..bfa1cf08c 100644
--- a/lib/sqlalchemy/dialects/mssql/base.py
+++ b/lib/sqlalchemy/dialects/mssql/base.py
@@ -1784,7 +1784,7 @@ class MSSQLCompiler(compiler.SQLCompiler):
     def limit_clause(self, cs, **kwargs):
         return ""
 
-    def _check_can_use_fetch_like(self, select):
+    def _check_can_use_fetch_limit(self, select):
         # to use ROW_NUMBER(), an ORDER BY is required.
         # OFFSET are FETCH are options of the ORDER BY clause
         if not select._order_by_clause.clauses:
@@ -1810,7 +1810,7 @@ class MSSQLCompiler(compiler.SQLCompiler):
         """
 
         if self.dialect._supports_offset_fetch and not self._use_top(select):
-            self._check_can_use_fetch_like(select)
+            self._check_can_use_fetch_limit(select)
 
             text = ""
 
@@ -1850,7 +1850,7 @@ class MSSQLCompiler(compiler.SQLCompiler):
             and not self._use_top(select)
             and not getattr(select, "_mssql_visit", None)
         ):
-            self._check_can_use_fetch_like(select)
+            self._check_can_use_fetch_limit(select)
 
             _order_by_clauses = [
                 sql_util.unwrap_label_reference(elem)
@@ -2031,7 +2031,10 @@ class MSSQLCompiler(compiler.SQLCompiler):
         if (
             self.is_subquery()
             and not select._limit
-            and (not select._offset or not self.dialect._supports_offset_fetch)
+            and (
+                select._offset is None
+                or not self.dialect._supports_offset_fetch
+            )
         ):
             # avoid processing the order by clause if we won't end up
             # using it, because we don't want all the bind params tacked
diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py
index 88b101811..46fcbbbe1 100644
--- a/lib/sqlalchemy/dialects/oracle/base.py
+++ b/lib/sqlalchemy/dialects/oracle/base.py
@@ -561,7 +561,6 @@ from ...types import CHAR
 from ...types import CLOB
 from ...types import FLOAT
 from ...types import INTEGER
-from ...types import Integer
 from ...types import NCHAR
 from ...types import NVARCHAR
 from ...types import TIMESTAMP
@@ -1092,6 +1091,13 @@ class OracleCompiler(compiler.SQLCompiler):
             ):
                 limit_clause = select._limit_clause
                 offset_clause = select._offset_clause
+
+                if select._simple_int_clause(limit_clause):
+                    limit_clause = limit_clause._render_literal_execute()
+
+                if select._simple_int_clause(offset_clause):
+                    offset_clause = offset_clause._render_literal_execute()
+
                 # currently using form at:
                 # https://blogs.oracle.com/oraclemagazine/\
                 # on-rownum-and-limiting-results
@@ -1120,22 +1126,17 @@ class OracleCompiler(compiler.SQLCompiler):
                         is not None
                     ]
                 )
+
                 if (
                     limit_clause is not None
                     and self.dialect.optimize_limits
                     and select._simple_int_clause(limit_clause)
                 ):
-                    param = sql.bindparam(
-                        "_ora_frow",
-                        select._limit,
-                        type_=Integer,
-                        literal_execute=True,
-                        unique=True,
-                    )
                     limitselect = limitselect.prefix_with(
                         expression.text(
-                            "/*+ FIRST_ROWS(:_ora_frow) */"
-                        ).bindparams(param)
+                            "/*+ FIRST_ROWS(%s) */"
+                            % self.process(limit_clause, **kwargs)
+                        )
                     )
 
                 limitselect._oracle_visit = True
@@ -1155,19 +1156,14 @@ class OracleCompiler(compiler.SQLCompiler):
                         offset_clause is None
                         or select._simple_int_clause(offset_clause)
                     ):
-                        max_row = select._limit
+                        max_row = limit_clause
 
                         if offset_clause is not None:
-                            max_row += select._offset
-                        max_row = sql.bindparam(
-                            None,
-                            max_row,
-                            type_=Integer,
-                            literal_execute=True,
-                            unique=True,
-                        )
+                            max_row = max_row + offset_clause
+
                     else:
                         max_row = limit_clause
+
                         if offset_clause is not None:
                             max_row = max_row + offset_clause
                     limitselect = limitselect.where(
@@ -1214,15 +1210,6 @@ class OracleCompiler(compiler.SQLCompiler):
                             adapter.traverse(elem) for elem in for_update.of
                         ]
 
-                    if select._simple_int_clause(offset_clause):
-                        offset_clause = sql.bindparam(
-                            None,
-                            select._offset,
-                            Integer,
-                            literal_execute=True,
-                            unique=True,
-                        )
-
                     offsetselect = offsetselect.where(
                         sql.literal_column("ora_rn") > offset_clause
                     )
diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py
index a2e5780f8..189deec41 100644
--- a/lib/sqlalchemy/sql/selectable.py
+++ b/lib/sqlalchemy/sql/selectable.py
@@ -69,6 +69,14 @@ class _OffsetLimitParam(BindParameter):
     def _limit_offset_value(self):
         return self.effective_value
 
+    def _render_literal_execute(self):
+        return _OffsetLimitParam(
+            self.key,
+            self.value,
+            type_=self.type,
+            literal_execute=True,
+        )
+
 
 @util.deprecated(
     "1.4",
diff --git a/lib/sqlalchemy/testing/suite/test_select.py b/lib/sqlalchemy/testing/suite/test_select.py
index c6dacd423..c519154f6 100644
--- a/lib/sqlalchemy/testing/suite/test_select.py
+++ b/lib/sqlalchemy/testing/suite/test_select.py
@@ -194,61 +194,116 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
             ],
         )
 
-    def _assert_result(self, select, result, params=()):
-        with config.db.connect() as conn:
-            eq_(conn.execute(select, params).fetchall(), result)
+    def _assert_result(
+        self, connection, select, result, params=(), set_=False
+    ):
+        if set_:
+            query_res = connection.execute(select, params).fetchall()
+            eq_(len(query_res), len(result))
+            eq_(set(query_res), set(result))
+
+        else:
+            eq_(connection.execute(select, params).fetchall(), result)
 
     def _assert_result_str(self, select, result, params=()):
         conn = config.db.connect(close_with_result=True)
         eq_(conn.exec_driver_sql(select, params).fetchall(), result)
 
-    def test_simple_limit(self):
+    def test_simple_limit(self, connection):
         table = self.tables.some_table
+        stmt = select(table).order_by(table.c.id)
         self._assert_result(
-            select(table).order_by(table.c.id).limit(2),
+            connection,
+            stmt.limit(2),
             [(1, 1, 2), (2, 2, 3)],
         )
+        self._assert_result(
+            connection,
+            stmt.limit(3),
+            [(1, 1, 2), (2, 2, 3), (3, 3, 4)],
+        )
 
     @testing.requires.fetch_first
-    def test_simple_fetch(self):
+    def test_simple_fetch(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).fetch(2),
             [(1, 1, 2), (2, 2, 3)],
         )
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).fetch(3),
+            [(1, 1, 2), (2, 2, 3), (3, 3, 4)],
+        )
 
     @testing.requires.offset
-    def test_simple_offset(self):
+    def test_simple_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).offset(2),
             [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
         )
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).offset(3),
+            [(4, 4, 5), (5, 4, 6)],
+        )
 
     @testing.requires.offset
-    def test_simple_limit_offset(self):
+    def test_simple_limit_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).limit(2).offset(1),
             [(2, 2, 3), (3, 3, 4)],
         )
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).limit(3).offset(2),
+            [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
+        )
 
     @testing.requires.fetch_first
-    def test_simple_fetch_offset(self):
+    def test_simple_fetch_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).fetch(2).offset(1),
             [(2, 2, 3), (3, 3, 4)],
         )
 
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).fetch(3).offset(2),
+            [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
+        )
+
     @testing.requires.fetch_no_order_by
-    def test_fetch_offset_no_order(self):
+    def test_fetch_offset_no_order(self, connection):
         table = self.tables.some_table
-        with config.db.connect() as conn:
-            eq_(
-                set(conn.execute(select(table).fetch(10))),
-                set([(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 4, 5), (5, 4, 6)]),
-            )
+        self._assert_result(
+            connection,
+            select(table).fetch(10),
+            [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 4, 5), (5, 4, 6)],
+            set_=True,
+        )
+
+    @testing.requires.offset
+    def test_simple_offset_zero(self, connection):
+        table = self.tables.some_table
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).offset(0),
+            [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 4, 5), (5, 4, 6)],
+        )
+
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).offset(1),
+            [(2, 2, 3), (3, 3, 4), (4, 4, 5), (5, 4, 6)],
+        )
 
     @testing.requires.offset
     def test_limit_offset_nobinds(self):
@@ -277,27 +332,44 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
         self._assert_result_str(sql, [(2, 2, 3), (3, 3, 4)])
 
     @testing.requires.bound_limit_offset
-    def test_bound_limit(self):
+    def test_bound_limit(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).limit(bindparam("l")),
             [(1, 1, 2), (2, 2, 3)],
             params={"l": 2},
         )
 
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).limit(bindparam("l")),
+            [(1, 1, 2), (2, 2, 3), (3, 3, 4)],
+            params={"l": 3},
+        )
+
     @testing.requires.bound_limit_offset
-    def test_bound_offset(self):
+    def test_bound_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).offset(bindparam("o")),
             [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
             params={"o": 2},
         )
 
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.id).offset(bindparam("o")),
+            [(2, 2, 3), (3, 3, 4), (4, 4, 5), (5, 4, 6)],
+            params={"o": 1},
+        )
+
     @testing.requires.bound_limit_offset
-    def test_bound_limit_offset(self):
+    def test_bound_limit_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .limit(bindparam("l"))
@@ -306,10 +378,21 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
             params={"l": 2, "o": 1},
         )
 
+        self._assert_result(
+            connection,
+            select(table)
+            .order_by(table.c.id)
+            .limit(bindparam("l"))
+            .offset(bindparam("o")),
+            [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
+            params={"l": 3, "o": 2},
+        )
+
     @testing.requires.fetch_first
-    def test_bound_fetch_offset(self):
+    def test_bound_fetch_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .fetch(bindparam("f"))
@@ -318,10 +401,21 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
             params={"f": 2, "o": 1},
         )
 
+        self._assert_result(
+            connection,
+            select(table)
+            .order_by(table.c.id)
+            .fetch(bindparam("f"))
+            .offset(bindparam("o")),
+            [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
+            params={"f": 3, "o": 2},
+        )
+
     @testing.requires.sql_expression_limit_offset
-    def test_expr_offset(self):
+    def test_expr_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .offset(literal_column("1") + literal_column("2")),
@@ -329,9 +423,10 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
         )
 
     @testing.requires.sql_expression_limit_offset
-    def test_expr_limit(self):
+    def test_expr_limit(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .limit(literal_column("1") + literal_column("2")),
@@ -339,9 +434,10 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
         )
 
     @testing.requires.sql_expression_limit_offset
-    def test_expr_limit_offset(self):
+    def test_expr_limit_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .limit(literal_column("1") + literal_column("1"))
@@ -350,9 +446,10 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
         )
 
     @testing.requires.fetch_first
-    def test_expr_fetch_offset(self):
+    def test_expr_fetch_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .fetch(literal_column("1") + literal_column("1"))
@@ -361,9 +458,10 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
         )
 
     @testing.requires.sql_expression_limit_offset
-    def test_simple_limit_expr_offset(self):
+    def test_simple_limit_expr_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .limit(2)
@@ -371,10 +469,20 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
             [(3, 3, 4), (4, 4, 5)],
         )
 
+        self._assert_result(
+            connection,
+            select(table)
+            .order_by(table.c.id)
+            .limit(3)
+            .offset(literal_column("1") + literal_column("1")),
+            [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
+        )
+
     @testing.requires.sql_expression_limit_offset
-    def test_expr_limit_simple_offset(self):
+    def test_expr_limit_simple_offset(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .limit(literal_column("1") + literal_column("1"))
@@ -382,40 +490,51 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
             [(3, 3, 4), (4, 4, 5)],
         )
 
+        self._assert_result(
+            connection,
+            select(table)
+            .order_by(table.c.id)
+            .limit(literal_column("1") + literal_column("1"))
+            .offset(1),
+            [(2, 2, 3), (3, 3, 4)],
+        )
+
     @testing.requires.fetch_ties
-    def test_simple_fetch_ties(self):
+    def test_simple_fetch_ties(self, connection):
         table = self.tables.some_table
-        with config.db.connect() as conn:
-            eq_(
-                set(
-                    conn.execute(
-                        select(table)
-                        .order_by(table.c.x.desc())
-                        .fetch(1, with_ties=True)
-                    )
-                ),
-                set([(4, 4, 5), (5, 4, 6)]),
-            )
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.x.desc()).fetch(1, with_ties=True),
+            [(4, 4, 5), (5, 4, 6)],
+            set_=True,
+        )
+
+        self._assert_result(
+            connection,
+            select(table).order_by(table.c.x.desc()).fetch(3, with_ties=True),
+            [(3, 3, 4), (4, 4, 5), (5, 4, 6)],
+            set_=True,
+        )
 
     @testing.requires.fetch_ties
     @testing.requires.fetch_offset_with_options
-    def test_fetch_offset_ties(self):
+    def test_fetch_offset_ties(self, connection):
         table = self.tables.some_table
-        with config.db.connect() as conn:
-            fa = conn.execute(
-                select(table)
-                .order_by(table.c.x)
-                .fetch(2, with_ties=True)
-                .offset(2)
-            ).fetchall()
-            eq_(fa[0], (3, 3, 4))
-            eq_(set(fa), set([(3, 3, 4), (4, 4, 5), (5, 4, 6)]))
+        fa = connection.execute(
+            select(table)
+            .order_by(table.c.x)
+            .fetch(2, with_ties=True)
+            .offset(2)
+        ).fetchall()
+        eq_(fa[0], (3, 3, 4))
+        eq_(set(fa), set([(3, 3, 4), (4, 4, 5), (5, 4, 6)]))
 
     @testing.requires.fetch_ties
     @testing.requires.fetch_offset_with_options
-    def test_fetch_offset_ties_exact_number(self):
+    def test_fetch_offset_ties_exact_number(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.x)
             .fetch(2, with_ties=True)
@@ -423,19 +542,30 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
             [(2, 2, 3), (3, 3, 4)],
         )
 
+        self._assert_result(
+            connection,
+            select(table)
+            .order_by(table.c.x)
+            .fetch(3, with_ties=True)
+            .offset(3),
+            [(4, 4, 5), (5, 4, 6)],
+        )
+
     @testing.requires.fetch_percent
-    def test_simple_fetch_percent(self):
+    def test_simple_fetch_percent(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table).order_by(table.c.id).fetch(20, percent=True),
             [(1, 1, 2)],
         )
 
     @testing.requires.fetch_percent
     @testing.requires.fetch_offset_with_options
-    def test_fetch_offset_percent(self):
+    def test_fetch_offset_percent(self, connection):
         table = self.tables.some_table
         self._assert_result(
+            connection,
             select(table)
             .order_by(table.c.id)
             .fetch(40, percent=True)
@@ -445,32 +575,30 @@ class FetchLimitOffsetTest(fixtures.TablesTest):
 
     @testing.requires.fetch_ties
     @testing.requires.fetch_percent
-    def test_simple_fetch_percent_ties(self):
+    def test_simple_fetch_percent_ties(self, connection):
         table = self.tables.some_table
-        with config.db.connect() as conn:
-            fa = conn.execute(
-                select(table)
-                .order_by(table.c.x.desc())
-                .fetch(20, percent=True, with_ties=True)
-            ).fetchall()
-
-        eq_(len(fa), 2)
-        eq_(set(fa), set([(4, 4, 5), (5, 4, 6)]))
+        self._assert_result(
+            connection,
+            select(table)
+            .order_by(table.c.x.desc())
+            .fetch(20, percent=True, with_ties=True),
+            [(4, 4, 5), (5, 4, 6)],
+            set_=True,
+        )
 
     @testing.requires.fetch_ties
     @testing.requires.fetch_percent
     @testing.requires.fetch_offset_with_options
-    def test_fetch_offset_percent_ties(self):
+    def test_fetch_offset_percent_ties(self, connection):
         table = self.tables.some_table
-        with config.db.connect() as conn:
-            fa = conn.execute(
-                select(table)
-                .order_by(table.c.x)
-                .fetch(40, percent=True, with_ties=True)
-                .offset(2)
-            ).fetchall()
-            eq_(fa[0], (3, 3, 4))
-            eq_(set(fa), set([(3, 3, 4), (4, 4, 5), (5, 4, 6)]))
+        fa = connection.execute(
+            select(table)
+            .order_by(table.c.x)
+            .fetch(40, percent=True, with_ties=True)
+            .offset(2)
+        ).fetchall()
+        eq_(fa[0], (3, 3, 4))
+        eq_(set(fa), set([(3, 3, 4), (4, 4, 5), (5, 4, 6)]))
 
 
 class JoinTest(fixtures.TablesTest):