Replace c extension with cython versions.workflow_test_cython

Re-implement c version immutabledict / processors / resultproxy / utils with cython.
Performance is in general in par or better than the c version
Added a collection module that has cython version of OrderedSet and IdentitySet

Added a new test/perf file to compare the implementations.
Run ``python test/perf/compiled_extensions.py all`` to execute the comparison test.

See results here: https://docs.google.com/document/d/1nOcDGojHRtXEkuy4vNXcW_XOJd9gqKhSeALGG3kYr6A/edit?usp=sharing

Fixes: #7256
Change-Id: I2930ef1894b5048210384728118e586e813f6a76
Signed-off-by: Federico Caselli <cfederico87@gmail.com>

2 files changed, 179 insertions, 29 deletions

diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py
index fb4fd02a1..59ebc87e2 100644
--- a/test/engine/test_execute.py
+++ b/test/engine/test_execute.py
@@ -119,7 +119,7 @@ class ExecuteTest(fixtures.TablesTest):
             tsa.exc.ArgumentError,
             "List argument must consist only of tuples or dictionaries",
             connection.exec_driver_sql,
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             [2, "fred"],
         )
 
@@ -127,7 +127,7 @@ class ExecuteTest(fixtures.TablesTest):
             tsa.exc.ArgumentError,
             "List argument must consist only of tuples or dictionaries",
             connection.exec_driver_sql,
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             [[3, "ed"], [4, "horse"]],
         )
 
@@ -159,23 +159,23 @@ class ExecuteTest(fixtures.TablesTest):
     def test_raw_qmark(self, connection):
         conn = connection
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             (1, "jack"),
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             (2, "fred"),
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             [(3, "ed"), (4, "horse")],
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             [(5, "barney"), (6, "donkey")],
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (?, ?)",
+            "insert into users (user_id, user_name) values (?, ?)",
             (7, "sally"),
         )
         res = conn.exec_driver_sql("select * from users order by user_id")
@@ -198,15 +198,15 @@ class ExecuteTest(fixtures.TablesTest):
     def test_raw_sprintf(self, connection):
         conn = connection
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (%s, %s)",
+            "insert into users (user_id, user_name) values (%s, %s)",
             (1, "jack"),
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (%s, %s)",
+            "insert into users (user_id, user_name) values (%s, %s)",
             [(2, "ed"), (3, "horse")],
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (%s, %s)",
+            "insert into users (user_id, user_name) values (%s, %s)",
             (4, "sally"),
         )
         conn.exec_driver_sql("insert into users (user_id) values (%s)", (5,))
@@ -254,15 +254,15 @@ class ExecuteTest(fixtures.TablesTest):
     def test_raw_named(self, connection):
         conn = connection
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (:id, :name)",
+            "insert into users (user_id, user_name) values (:id, :name)",
             {"id": 1, "name": "jack"},
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (:id, :name)",
+            "insert into users (user_id, user_name) values (:id, :name)",
             [{"id": 2, "name": "ed"}, {"id": 3, "name": "horse"}],
         )
         conn.exec_driver_sql(
-            "insert into users (user_id, user_name) " "values (:id, :name)",
+            "insert into users (user_id, user_name) values (:id, :name)",
             {"id": 4, "name": "sally"},
         )
         res = conn.exec_driver_sql("select * from users order by user_id")
@@ -518,7 +518,7 @@ class ExecuteTest(fixtures.TablesTest):
     )
     @testing.fails_on(
         "oracle+cx_oracle",
-        "cx_oracle exception seems to be having " "some issue with pickling",
+        "cx_oracle exception seems to be having some issue with pickling",
     )
     def test_stmt_exception_pickleable_plus_dbapi(self):
         raw = testing.db.raw_connection()
diff --git a/test/engine/test_processors.py b/test/engine/test_processors.py
index 943ae32f0..392632327 100644
--- a/test/engine/test_processors.py
+++ b/test/engine/test_processors.py
@@ -1,6 +1,11 @@
+from types import MappingProxyType
+
+from sqlalchemy import exc
 from sqlalchemy.testing import assert_raises_message
 from sqlalchemy.testing import eq_
+from sqlalchemy.testing import expect_raises_message
 from sqlalchemy.testing import fixtures
+from sqlalchemy.util import immutabledict
 
 
 class _BooleanProcessorTest(fixtures.TestBase):
@@ -20,14 +25,14 @@ class _BooleanProcessorTest(fixtures.TestBase):
         eq_(self.module.int_to_boolean(-4), True)
 
 
-class CBooleanProcessorTest(_BooleanProcessorTest):
+class CyBooleanProcessorTest(_BooleanProcessorTest):
     __requires__ = ("cextensions",)
 
     @classmethod
     def setup_test_class(cls):
-        from sqlalchemy import cprocessors
+        from sqlalchemy.cyextension import processors
 
-        cls.module = cprocessors
+        cls.module = processors
 
 
 class _DateProcessorTest(fixtures.TestBase):
@@ -83,23 +88,168 @@ class _DateProcessorTest(fixtures.TestBase):
 class PyDateProcessorTest(_DateProcessorTest):
     @classmethod
     def setup_test_class(cls):
-        from sqlalchemy import processors
-
-        cls.module = type(
-            "util",
-            (object,),
-            dict(
-                (k, staticmethod(v))
-                for k, v in list(processors.py_fallback().items())
-            ),
+        from sqlalchemy.engine import _py_processors
+
+        cls.module = _py_processors
+
+
+class CyDateProcessorTest(_DateProcessorTest):
+    __requires__ = ("cextensions",)
+
+    @classmethod
+    def setup_test_class(cls):
+        from sqlalchemy.cyextension import processors
+
+        cls.module = processors
+
+
+class _DistillArgsTest(fixtures.TestBase):
+    def test_distill_20_none(self):
+        eq_(self.module._distill_params_20(None), ())
+
+    def test_distill_20_empty_sequence(self):
+        eq_(self.module._distill_params_20(()), ())
+        eq_(self.module._distill_params_20([]), [])
+
+    def test_distill_20_sequence_sequence(self):
+        eq_(self.module._distill_params_20(((1, 2, 3),)), ((1, 2, 3),))
+        eq_(self.module._distill_params_20([(1, 2, 3)]), [(1, 2, 3)])
+
+        eq_(self.module._distill_params_20(((1, 2), (2, 3))), ((1, 2), (2, 3)))
+        eq_(self.module._distill_params_20([(1, 2), (2, 3)]), [(1, 2), (2, 3)])
+
+    def test_distill_20_sequence_dict(self):
+        eq_(self.module._distill_params_20(({"a": 1},)), ({"a": 1},))
+        eq_(
+            self.module._distill_params_20([{"a": 1}, {"a": 2}]),
+            [{"a": 1}, {"a": 2}],
         )
+        eq_(
+            self.module._distill_params_20((MappingProxyType({"a": 1}),)),
+            (MappingProxyType({"a": 1}),),
+        )
+
+    def test_distill_20_sequence_error(self):
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_params_20((1, 2, 3))
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_params_20(([1, 2, 3],))
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_params_20([1, 2, 3])
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_params_20(["a", "b"])
+
+    def test_distill_20_dict(self):
+        eq_(self.module._distill_params_20({"foo": "bar"}), [{"foo": "bar"}])
+        eq_(
+            self.module._distill_params_20(immutabledict({"foo": "bar"})),
+            [immutabledict({"foo": "bar"})],
+        )
+        eq_(
+            self.module._distill_params_20(MappingProxyType({"foo": "bar"})),
+            [MappingProxyType({"foo": "bar"})],
+        )
+
+    def test_distill_20_error(self):
+        with expect_raises_message(
+            exc.ArgumentError, "mapping or list expected for parameters"
+        ):
+            self.module._distill_params_20("foo")
+        with expect_raises_message(
+            exc.ArgumentError, "mapping or list expected for parameters"
+        ):
+            self.module._distill_params_20(1)
+
+    def test_distill_raw_none(self):
+        eq_(self.module._distill_raw_params(None), ())
+
+    def test_distill_raw_empty_list(self):
+        eq_(self.module._distill_raw_params([]), [])
+
+    def test_distill_raw_list_sequence(self):
+        eq_(self.module._distill_raw_params([(1, 2, 3)]), [(1, 2, 3)])
+        eq_(
+            self.module._distill_raw_params([(1, 2), (2, 3)]), [(1, 2), (2, 3)]
+        )
+
+    def test_distill_raw_list_dict(self):
+        eq_(
+            self.module._distill_raw_params([{"a": 1}, {"a": 2}]),
+            [{"a": 1}, {"a": 2}],
+        )
+        eq_(
+            self.module._distill_raw_params([MappingProxyType({"a": 1})]),
+            [MappingProxyType({"a": 1})],
+        )
+
+    def test_distill_raw_sequence_error(self):
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_raw_params([1, 2, 3])
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_raw_params([[1, 2, 3]])
+        with expect_raises_message(
+            exc.ArgumentError,
+            "List argument must consist only of tuples or dictionaries",
+        ):
+            self.module._distill_raw_params(["a", "b"])
+
+    def test_distill_raw_tuple(self):
+        eq_(self.module._distill_raw_params(()), [()])
+        eq_(self.module._distill_raw_params((1, 2, 3)), [(1, 2, 3)])
+
+    def test_distill_raw_dict(self):
+        eq_(self.module._distill_raw_params({"foo": "bar"}), [{"foo": "bar"}])
+        eq_(
+            self.module._distill_raw_params(immutabledict({"foo": "bar"})),
+            [immutabledict({"foo": "bar"})],
+        )
+        eq_(
+            self.module._distill_raw_params(MappingProxyType({"foo": "bar"})),
+            [MappingProxyType({"foo": "bar"})],
+        )
+
+    def test_distill_raw_error(self):
+        with expect_raises_message(
+            exc.ArgumentError, "mapping or sequence expected for parameters"
+        ):
+            self.module._distill_raw_params("foo")
+        with expect_raises_message(
+            exc.ArgumentError, "mapping or sequence expected for parameters"
+        ):
+            self.module._distill_raw_params(1)
+
+
+class PyDistillArgsTest(_DistillArgsTest):
+    @classmethod
+    def setup_test_class(cls):
+        from sqlalchemy.engine import _py_util
+
+        cls.module = _py_util
 
 
-class CDateProcessorTest(_DateProcessorTest):
+class CyDistillArgsTest(_DistillArgsTest):
     __requires__ = ("cextensions",)
 
     @classmethod
     def setup_test_class(cls):
-        from sqlalchemy import cprocessors
+        from sqlalchemy.cyextension import util
 
-        cls.module = cprocessors
+        cls.module = util