Fix tests failing for SQLite file databases; repair provisioning

1. ensure provision.py loads dialect implementations when running
reap_dbs.py.   Reapers haven't been working since
598f2f7e557073f29563d4d567f43931fc03013f .

2. add some exclusion rules to allow the sqlite_file target to work;
add to tox.

3. add reap dbs target for SQLite, repair SQLite drop_db routine
which also wasn't doing the right thing for memory databases
etc.

4. Fix logging in provision files, as the main provision logger
is the one that's enabled by reap_dbs and maybe others, have all
the provision files use the provision logger.

Fixes: #5180
Fixes: #5168

Change-Id: Ibc1b0106394d20f5bcf847f37b09d185f26ac9b5

5 files changed, 36 insertions, 19 deletions

diff --git a/lib/sqlalchemy/dialects/mssql/provision.py b/lib/sqlalchemy/dialects/mssql/provision.py
index 558ad7a7a..a10043db1 100644
--- a/lib/sqlalchemy/dialects/mssql/provision.py
+++ b/lib/sqlalchemy/dialects/mssql/provision.py
@@ -1,16 +1,12 @@
-import logging
-
 from ... import create_engine
 from ... import exc
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
+from ...testing.provision import log
 from ...testing.provision import run_reap_dbs
 from ...testing.provision import update_db_opts
 
 
-log = logging.getLogger(__name__)
-
-
 @update_db_opts.for_db("mssql")
 def _mssql_update_db_opts(db_url, db_opts):
     db_opts["legacy_schema_aliasing"] = False
diff --git a/lib/sqlalchemy/dialects/oracle/provision.py b/lib/sqlalchemy/dialects/oracle/provision.py
index 8fc2ac235..7901eb4e8 100644
--- a/lib/sqlalchemy/dialects/oracle/provision.py
+++ b/lib/sqlalchemy/dialects/oracle/provision.py
@@ -1,5 +1,3 @@
-import logging
-
 from ... import create_engine
 from ... import exc
 from ...engine import url as sa_url
@@ -7,14 +5,12 @@ from ...testing.provision import configure_follower
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
 from ...testing.provision import follower_url_from_main
+from ...testing.provision import log
 from ...testing.provision import run_reap_dbs
 from ...testing.provision import temp_table_keyword_args
 from ...testing.provision import update_db_opts
 
 
-log = logging.getLogger(__name__)
-
-
 @create_db.for_db("oracle")
 def _oracle_create_db(cfg, eng, ident):
     # NOTE: make sure you've run "ALTER DATABASE default tablespace users" or
diff --git a/lib/sqlalchemy/dialects/postgresql/provision.py b/lib/sqlalchemy/dialects/postgresql/provision.py
index 404da93a8..c8f83d2be 100644
--- a/lib/sqlalchemy/dialects/postgresql/provision.py
+++ b/lib/sqlalchemy/dialects/postgresql/provision.py
@@ -1,16 +1,13 @@
-import logging
 import time
 
 from ... import exc
 from ... import text
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
+from ...testing.provision import log
 from ...testing.provision import temp_table_keyword_args
 
 
-log = logging.getLogger(__name__)
-
-
 @create_db.for_db("postgresql")
 def _pg_create_db(cfg, eng, ident):
     template_db = cfg.options.postgresql_templatedb
diff --git a/lib/sqlalchemy/dialects/sqlite/provision.py b/lib/sqlalchemy/dialects/sqlite/provision.py
index d4a5ae93f..ce20ed991 100644
--- a/lib/sqlalchemy/dialects/sqlite/provision.py
+++ b/lib/sqlalchemy/dialects/sqlite/provision.py
@@ -4,7 +4,9 @@ from ...engine import url as sa_url
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
 from ...testing.provision import follower_url_from_main
+from ...testing.provision import log
 from ...testing.provision import post_configure_engine
+from ...testing.provision import run_reap_dbs
 from ...testing.provision import temp_table_keyword_args
 
 
@@ -26,6 +28,11 @@ def _sqlite_post_configure_engine(url, engine, follower_ident):
         # use file DBs in all cases, memory acts kind of strangely
         # as an attached
         if not follower_ident:
+            # note this test_schema.db gets created for all test runs.
+            # there's not any dedicated cleanup step for it.  it in some
+            # ways corresponds to the "test.test_schema" schema that's
+            # expected to be already present, so for now it just stays
+            # in a given checkout directory.
             dbapi_connection.execute(
                 'ATTACH DATABASE "test_schema.db" AS test_schema'
             )
@@ -43,12 +50,26 @@ def _sqlite_create_db(cfg, eng, ident):
 
 @drop_db.for_db("sqlite")
 def _sqlite_drop_db(cfg, eng, ident):
-    if ident:
-        os.remove("%s_test_schema.db" % ident)
-    else:
-        os.remove("%s.db" % ident)
+    for path in ["%s.db" % ident, "%s_test_schema.db" % ident]:
+        if os.path.exists(path):
+            log.info("deleting SQLite database file: %s" % path)
+            os.remove(path)
 
 
 @temp_table_keyword_args.for_db("sqlite")
 def _sqlite_temp_table_keyword_args(cfg, eng):
     return {"prefixes": ["TEMPORARY"]}
+
+
+@run_reap_dbs.for_db("sqlite")
+def _reap_sqlite_dbs(url, idents):
+    log.info("db reaper connecting to %r", url)
+
+    log.info("identifiers in file: %s", ", ".join(idents))
+    for ident in idents:
+        # we don't have a config so we can't call _sqlite_drop_db due to the
+        # decorator
+        for path in ["%s.db" % ident, "%s_test_schema.db" % ident]:
+            if os.path.exists(path):
+                log.info("deleting SQLite database file: %s" % path)
+                os.remove(path)
diff --git a/lib/sqlalchemy/dialects/sqlite/pysqlite.py b/lib/sqlalchemy/dialects/sqlite/pysqlite.py
index 0585e69ad..4485631ce 100644
--- a/lib/sqlalchemy/dialects/sqlite/pysqlite.py
+++ b/lib/sqlalchemy/dialects/sqlite/pysqlite.py
@@ -421,8 +421,15 @@ class SQLiteDialect_pysqlite(SQLiteDialect):
         return sqlite
 
     @classmethod
-    def get_pool_class(cls, url):
+    def _is_url_file_db(cls, url):
         if url.database and url.database != ":memory:":
+            return True
+        else:
+            return False
+
+    @classmethod
+    def get_pool_class(cls, url):
+        if cls._is_url_file_db(url):
             return pool.NullPool
         else:
             return pool.SingletonThreadPool