summaryrefslogtreecommitdiff
path: root/examples/performance
diff options
context:
space:
mode:
authorMike Bayer <mike_mp@zzzcomputing.com>2019-01-06 01:14:26 -0500
committermike bayer <mike_mp@zzzcomputing.com>2019-01-06 17:34:50 +0000
commit1e1a38e7801f410f244e4bbb44ec795ae152e04e (patch)
tree28e725c5c8188bd0cfd133d1e268dbca9b524978 /examples/performance
parent404e69426b05a82d905cbb3ad33adafccddb00dd (diff)
downloadsqlalchemy-1e1a38e7801f410f244e4bbb44ec795ae152e04e.tar.gz
Run black -l 79 against all source files
This is a straight reformat run using black as is, with no edits applied at all. The black run will format code consistently, however in some cases that are prevalent in SQLAlchemy code it produces too-long lines. The too-long lines will be resolved in the following commit that will resolve all remaining flake8 issues including shadowed builtins, long lines, import order, unused imports, duplicate imports, and docstring issues. Change-Id: I7eda77fed3d8e73df84b3651fd6cfcfe858d4dc9
Diffstat (limited to 'examples/performance')
-rw-r--r--examples/performance/__init__.py67
-rw-r--r--examples/performance/__main__.py3
-rw-r--r--examples/performance/bulk_inserts.py115
-rw-r--r--examples/performance/bulk_updates.py23
-rw-r--r--examples/performance/large_resultsets.py50
-rw-r--r--examples/performance/short_selects.py60
-rw-r--r--examples/performance/single_inserts.py66
7 files changed, 226 insertions, 158 deletions
diff --git a/examples/performance/__init__.py b/examples/performance/__init__.py
index 6264ae9f7..b66199f3c 100644
--- a/examples/performance/__init__.py
+++ b/examples/performance/__init__.py
@@ -255,7 +255,8 @@ class Profiler(object):
def profile(cls, fn):
if cls.name is None:
raise ValueError(
- "Need to call Profile.init(<suitename>, <default_num>) first.")
+ "Need to call Profile.init(<suitename>, <default_num>) first."
+ )
cls.tests.append(fn)
return fn
@@ -270,7 +271,8 @@ class Profiler(object):
def setup_once(cls, fn):
if cls._setup_once is not None:
raise ValueError(
- "setup_once function already set to %s" % cls._setup_once)
+ "setup_once function already set to %s" % cls._setup_once
+ )
cls._setup_once = staticmethod(fn)
return fn
@@ -298,7 +300,7 @@ class Profiler(object):
finally:
pr.disable()
- stats = pstats.Stats(pr).sort_stats('cumulative')
+ stats = pstats.Stats(pr).sort_stats("cumulative")
self.stats.append(TestResult(self, fn, stats=stats))
return result
@@ -326,7 +328,8 @@ class Profiler(object):
if cls.name is None:
parser.add_argument(
- "name", choices=cls._suite_names(), help="suite to run")
+ "name", choices=cls._suite_names(), help="suite to run"
+ )
if len(sys.argv) > 1:
potential_name = sys.argv[1]
@@ -335,35 +338,44 @@ class Profiler(object):
except ImportError:
pass
- parser.add_argument(
- "--test", type=str,
- help="run specific test name"
- )
+ parser.add_argument("--test", type=str, help="run specific test name")
parser.add_argument(
- '--dburl', type=str, default="sqlite:///profile.db",
- help="database URL, default sqlite:///profile.db"
+ "--dburl",
+ type=str,
+ default="sqlite:///profile.db",
+ help="database URL, default sqlite:///profile.db",
)
parser.add_argument(
- '--num', type=int, default=cls.num,
+ "--num",
+ type=int,
+ default=cls.num,
help="Number of iterations/items/etc for tests; "
- "default is %d module-specific" % cls.num
+ "default is %d module-specific" % cls.num,
)
parser.add_argument(
- '--profile', action='store_true',
- help='run profiling and dump call counts')
+ "--profile",
+ action="store_true",
+ help="run profiling and dump call counts",
+ )
parser.add_argument(
- '--dump', action='store_true',
- help='dump full call profile (implies --profile)')
+ "--dump",
+ action="store_true",
+ help="dump full call profile (implies --profile)",
+ )
parser.add_argument(
- '--callers', action='store_true',
- help='print callers as well (implies --dump)')
+ "--callers",
+ action="store_true",
+ help="print callers as well (implies --dump)",
+ )
parser.add_argument(
- '--runsnake', action='store_true',
- help='invoke runsnakerun (implies --profile)')
+ "--runsnake",
+ action="store_true",
+ help="invoke runsnakerun (implies --profile)",
+ )
parser.add_argument(
- '--echo', action='store_true',
- help="Echo SQL output")
+ "--echo", action="store_true", help="Echo SQL output"
+ )
args = parser.parse_args()
args.dump = args.dump or args.callers
@@ -378,7 +390,7 @@ class Profiler(object):
def _suite_names(cls):
suites = []
for file_ in os.listdir(os.path.dirname(__file__)):
- match = re.match(r'^([a-z].*).py$', file_)
+ match = re.match(r"^([a-z].*).py$", file_)
if match:
suites.append(match.group(1))
return suites
@@ -398,7 +410,10 @@ class TestResult(object):
def _summary(self):
summary = "%s : %s (%d iterations)" % (
- self.test.__name__, self.test.__doc__, self.profile.num)
+ self.test.__name__,
+ self.test.__doc__,
+ self.profile.num,
+ )
if self.total_time:
summary += "; total time %f sec" % self.total_time
if self.stats:
@@ -412,7 +427,7 @@ class TestResult(object):
self._dump()
def _dump(self):
- self.stats.sort_stats('time', 'calls')
+ self.stats.sort_stats("time", "calls")
self.stats.print_stats()
if self.profile.callers:
self.stats.print_callers()
@@ -424,5 +439,3 @@ class TestResult(object):
os.system("runsnake %s" % filename)
finally:
os.remove(filename)
-
-
diff --git a/examples/performance/__main__.py b/examples/performance/__main__.py
index 5e05143bf..945458651 100644
--- a/examples/performance/__main__.py
+++ b/examples/performance/__main__.py
@@ -2,6 +2,5 @@
from . import Profiler
-if __name__ == '__main__':
+if __name__ == "__main__":
Profiler.main()
-
diff --git a/examples/performance/bulk_inserts.py b/examples/performance/bulk_inserts.py
index 9c3cff5b2..52f0f32e6 100644
--- a/examples/performance/bulk_inserts.py
+++ b/examples/performance/bulk_inserts.py
@@ -36,12 +36,15 @@ def test_flush_no_pk(n):
"""Individual INSERT statements via the ORM, calling upon last row id"""
session = Session(bind=engine)
for chunk in range(0, n, 1000):
- session.add_all([
- Customer(
- name='customer name %d' % i,
- description='customer description %d' % i)
- for i in range(chunk, chunk + 1000)
- ])
+ session.add_all(
+ [
+ Customer(
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ for i in range(chunk, chunk + 1000)
+ ]
+ )
session.flush()
session.commit()
@@ -50,13 +53,16 @@ def test_flush_no_pk(n):
def test_bulk_save_return_pks(n):
"""Individual INSERT statements in "bulk", but calling upon last row id"""
session = Session(bind=engine)
- session.bulk_save_objects([
- Customer(
- name='customer name %d' % i,
- description='customer description %d' % i
- )
- for i in range(n)
- ], return_defaults=True)
+ session.bulk_save_objects(
+ [
+ Customer(
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ for i in range(n)
+ ],
+ return_defaults=True,
+ )
session.commit()
@@ -65,13 +71,16 @@ def test_flush_pk_given(n):
"""Batched INSERT statements via the ORM, PKs already defined"""
session = Session(bind=engine)
for chunk in range(0, n, 1000):
- session.add_all([
- Customer(
- id=i + 1,
- name='customer name %d' % i,
- description='customer description %d' % i)
- for i in range(chunk, chunk + 1000)
- ])
+ session.add_all(
+ [
+ Customer(
+ id=i + 1,
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ for i in range(chunk, chunk + 1000)
+ ]
+ )
session.flush()
session.commit()
@@ -80,13 +89,15 @@ def test_flush_pk_given(n):
def test_bulk_save(n):
"""Batched INSERT statements via the ORM in "bulk", discarding PKs."""
session = Session(bind=engine)
- session.bulk_save_objects([
- Customer(
- name='customer name %d' % i,
- description='customer description %d' % i
- )
- for i in range(n)
- ])
+ session.bulk_save_objects(
+ [
+ Customer(
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ for i in range(n)
+ ]
+ )
session.commit()
@@ -94,13 +105,16 @@ def test_bulk_save(n):
def test_bulk_insert_mappings(n):
"""Batched INSERT statements via the ORM "bulk", using dictionaries."""
session = Session(bind=engine)
- session.bulk_insert_mappings(Customer, [
- dict(
- name='customer name %d' % i,
- description='customer description %d' % i
- )
- for i in range(n)
- ])
+ session.bulk_insert_mappings(
+ Customer,
+ [
+ dict(
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ for i in range(n)
+ ],
+ )
session.commit()
@@ -112,11 +126,12 @@ def test_core_insert(n):
Customer.__table__.insert(),
[
dict(
- name='customer name %d' % i,
- description='customer description %d' % i
+ name="customer name %d" % i,
+ description="customer description %d" % i,
)
for i in range(n)
- ])
+ ],
+ )
@Profiler.profile
@@ -125,30 +140,30 @@ def test_dbapi_raw(n):
conn = engine.pool._creator()
cursor = conn.cursor()
- compiled = Customer.__table__.insert().values(
- name=bindparam('name'),
- description=bindparam('description')).\
- compile(dialect=engine.dialect)
+ compiled = (
+ Customer.__table__.insert()
+ .values(name=bindparam("name"), description=bindparam("description"))
+ .compile(dialect=engine.dialect)
+ )
if compiled.positional:
args = (
- ('customer name %d' % i, 'customer description %d' % i)
- for i in range(n))
+ ("customer name %d" % i, "customer description %d" % i)
+ for i in range(n)
+ )
else:
args = (
dict(
- name='customer name %d' % i,
- description='customer description %d' % i
+ name="customer name %d" % i,
+ description="customer description %d" % i,
)
for i in range(n)
)
- cursor.executemany(
- str(compiled),
- list(args)
- )
+ cursor.executemany(str(compiled), list(args))
conn.commit()
conn.close()
-if __name__ == '__main__':
+
+if __name__ == "__main__":
Profiler.main()
diff --git a/examples/performance/bulk_updates.py b/examples/performance/bulk_updates.py
index 9522e4bf5..ebb700068 100644
--- a/examples/performance/bulk_updates.py
+++ b/examples/performance/bulk_updates.py
@@ -32,12 +32,16 @@ def setup_database(dburl, echo, num):
s = Session(engine)
for chunk in range(0, num, 10000):
- s.bulk_insert_mappings(Customer, [
- {
- 'name': 'customer name %d' % i,
- 'description': 'customer description %d' % i
- } for i in range(chunk, chunk + 10000)
- ])
+ s.bulk_insert_mappings(
+ Customer,
+ [
+ {
+ "name": "customer name %d" % i,
+ "description": "customer description %d" % i,
+ }
+ for i in range(chunk, chunk + 10000)
+ ],
+ )
s.commit()
@@ -46,8 +50,11 @@ def test_orm_flush(n):
"""UPDATE statements via the ORM flush process."""
session = Session(bind=engine)
for chunk in range(0, n, 1000):
- customers = session.query(Customer).\
- filter(Customer.id.between(chunk, chunk + 1000)).all()
+ customers = (
+ session.query(Customer)
+ .filter(Customer.id.between(chunk, chunk + 1000))
+ .all()
+ )
for customer in customers:
customer.description += "updated"
session.flush()
diff --git a/examples/performance/large_resultsets.py b/examples/performance/large_resultsets.py
index c13683040..ad1c23194 100644
--- a/examples/performance/large_resultsets.py
+++ b/examples/performance/large_resultsets.py
@@ -46,9 +46,12 @@ def setup_database(dburl, echo, num):
Customer.__table__.insert(),
params=[
{
- 'name': 'customer name %d' % i,
- 'description': 'customer description %d' % i
- } for i in range(chunk, chunk + 10000)])
+ "name": "customer name %d" % i,
+ "description": "customer description %d" % i,
+ }
+ for i in range(chunk, chunk + 10000)
+ ],
+ )
s.commit()
@@ -74,8 +77,9 @@ def test_orm_bundles(n):
"""Load lightweight "bundle" objects using the ORM."""
sess = Session(engine)
- bundle = Bundle('customer',
- Customer.id, Customer.name, Customer.description)
+ bundle = Bundle(
+ "customer", Customer.id, Customer.name, Customer.description
+ )
for row in sess.query(bundle).yield_per(10000).limit(n):
pass
@@ -85,9 +89,11 @@ def test_orm_columns(n):
"""Load individual columns into named tuples using the ORM."""
sess = Session(engine)
- for row in sess.query(
- Customer.id, Customer.name,
- Customer.description).yield_per(10000).limit(n):
+ for row in (
+ sess.query(Customer.id, Customer.name, Customer.description)
+ .yield_per(10000)
+ .limit(n)
+ ):
pass
@@ -98,7 +104,7 @@ def test_core_fetchall(n):
with engine.connect() as conn:
result = conn.execute(Customer.__table__.select().limit(n)).fetchall()
for row in result:
- data = row['id'], row['name'], row['description']
+ data = row["id"], row["name"], row["description"]
@Profiler.profile
@@ -106,14 +112,15 @@ def test_core_fetchmany_w_streaming(n):
"""Load Core result rows using fetchmany/streaming."""
with engine.connect() as conn:
- result = conn.execution_options(stream_results=True).\
- execute(Customer.__table__.select().limit(n))
+ result = conn.execution_options(stream_results=True).execute(
+ Customer.__table__.select().limit(n)
+ )
while True:
chunk = result.fetchmany(10000)
if not chunk:
break
for row in chunk:
- data = row['id'], row['name'], row['description']
+ data = row["id"], row["name"], row["description"]
@Profiler.profile
@@ -127,7 +134,7 @@ def test_core_fetchmany(n):
if not chunk:
break
for row in chunk:
- data = row['id'], row['name'], row['description']
+ data = row["id"], row["name"], row["description"]
@Profiler.profile
@@ -145,10 +152,13 @@ def test_dbapi_fetchall_no_object(n):
def _test_dbapi_raw(n, make_objects):
- compiled = Customer.__table__.select().limit(n).\
- compile(
- dialect=engine.dialect,
- compile_kwargs={"literal_binds": True})
+ compiled = (
+ Customer.__table__.select()
+ .limit(n)
+ .compile(
+ dialect=engine.dialect, compile_kwargs={"literal_binds": True}
+ )
+ )
if make_objects:
# because if you're going to roll your own, you're probably
@@ -170,7 +180,8 @@ def _test_dbapi_raw(n, make_objects):
for row in cursor.fetchall():
# ensure that we fully fetch!
customer = SimpleCustomer(
- id=row[0], name=row[1], description=row[2])
+ id=row[0], name=row[1], description=row[2]
+ )
else:
for row in cursor.fetchall():
# ensure that we fully fetch!
@@ -178,5 +189,6 @@ def _test_dbapi_raw(n, make_objects):
conn.close()
-if __name__ == '__main__':
+
+if __name__ == "__main__":
Profiler.main()
diff --git a/examples/performance/short_selects.py b/examples/performance/short_selects.py
index 6f64aa63e..4a8d401ad 100644
--- a/examples/performance/short_selects.py
+++ b/examples/performance/short_selects.py
@@ -6,8 +6,14 @@ record by primary key
from . import Profiler
from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import Column, Integer, String, create_engine, \
- bindparam, select
+from sqlalchemy import (
+ Column,
+ Integer,
+ String,
+ create_engine,
+ bindparam,
+ select,
+)
from sqlalchemy.orm import Session, deferred
from sqlalchemy.ext import baked
import random
@@ -29,6 +35,7 @@ class Customer(Base):
y = deferred(Column(Integer))
z = deferred(Column(Integer))
+
Profiler.init("short_selects", num=10000)
@@ -39,16 +46,20 @@ def setup_database(dburl, echo, num):
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
sess = Session(engine)
- sess.add_all([
- Customer(
- id=i, name='c%d' % i, description="c%d" % i,
- q=i * 10,
- p=i * 20,
- x=i * 30,
- y=i * 40,
- )
- for i in ids
- ])
+ sess.add_all(
+ [
+ Customer(
+ id=i,
+ name="c%d" % i,
+ description="c%d" % i,
+ q=i * 10,
+ p=i * 20,
+ x=i * 30,
+ y=i * 40,
+ )
+ for i in ids
+ ]
+ )
sess.commit()
@@ -65,9 +76,9 @@ def test_orm_query_cols_only(n):
"""test an ORM query of only the entity columns."""
session = Session(bind=engine)
for id_ in random.sample(ids, n):
- session.query(
- Customer.id, Customer.name, Customer.description
- ).filter(Customer.id == id_).one()
+ session.query(Customer.id, Customer.name, Customer.description).filter(
+ Customer.id == id_
+ ).one()
@Profiler.profile
@@ -77,7 +88,7 @@ def test_baked_query(n):
s = Session(bind=engine)
for id_ in random.sample(ids, n):
q = bakery(lambda s: s.query(Customer))
- q += lambda q: q.filter(Customer.id == bindparam('id'))
+ q += lambda q: q.filter(Customer.id == bindparam("id"))
q(s).params(id=id_).one()
@@ -88,9 +99,9 @@ def test_baked_query_cols_only(n):
s = Session(bind=engine)
for id_ in random.sample(ids, n):
q = bakery(
- lambda s: s.query(
- Customer.id, Customer.name, Customer.description))
- q += lambda q: q.filter(Customer.id == bindparam('id'))
+ lambda s: s.query(Customer.id, Customer.name, Customer.description)
+ )
+ q += lambda q: q.filter(Customer.id == bindparam("id"))
q(s).params(id=id_).one()
@@ -109,7 +120,7 @@ def test_core_new_stmt_each_time(n):
def test_core_reuse_stmt(n):
"""test core, reusing the same statement (but recompiling each time)."""
- stmt = select([Customer.__table__]).where(Customer.id == bindparam('id'))
+ stmt = select([Customer.__table__]).where(Customer.id == bindparam("id"))
with engine.connect() as conn:
for id_ in random.sample(ids, n):
@@ -122,13 +133,14 @@ def test_core_reuse_stmt_compiled_cache(n):
"""test core, reusing the same statement + compiled cache."""
compiled_cache = {}
- stmt = select([Customer.__table__]).where(Customer.id == bindparam('id'))
- with engine.connect().\
- execution_options(compiled_cache=compiled_cache) as conn:
+ stmt = select([Customer.__table__]).where(Customer.id == bindparam("id"))
+ with engine.connect().execution_options(
+ compiled_cache=compiled_cache
+ ) as conn:
for id_ in random.sample(ids, n):
row = conn.execute(stmt, id=id_).first()
tuple(row)
-if __name__ == '__main__':
+if __name__ == "__main__":
Profiler.main()
diff --git a/examples/performance/single_inserts.py b/examples/performance/single_inserts.py
index cfce90300..79e34dfe6 100644
--- a/examples/performance/single_inserts.py
+++ b/examples/performance/single_inserts.py
@@ -28,7 +28,7 @@ Profiler.init("single_inserts", num=10000)
def setup_database(dburl, echo, num):
global engine
engine = create_engine(dburl, echo=echo)
- if engine.dialect.name == 'sqlite':
+ if engine.dialect.name == "sqlite":
engine.pool = pool.StaticPool(creator=engine.pool._creator)
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
@@ -42,8 +42,9 @@ def test_orm_commit(n):
session = Session(bind=engine)
session.add(
Customer(
- name='customer name %d' % i,
- description='customer description %d' % i)
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
)
session.commit()
@@ -54,11 +55,14 @@ def test_bulk_save(n):
for i in range(n):
session = Session(bind=engine)
- session.bulk_save_objects([
- Customer(
- name='customer name %d' % i,
- description='customer description %d' % i
- )])
+ session.bulk_save_objects(
+ [
+ Customer(
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ ]
+ )
session.commit()
@@ -68,11 +72,15 @@ def test_bulk_insert_dictionaries(n):
for i in range(n):
session = Session(bind=engine)
- session.bulk_insert_mappings(Customer, [
- dict(
- name='customer name %d' % i,
- description='customer description %d' % i
- )])
+ session.bulk_insert_mappings(
+ Customer,
+ [
+ dict(
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ )
+ ],
+ )
session.commit()
@@ -85,9 +93,9 @@ def test_core(n):
conn.execute(
Customer.__table__.insert(),
dict(
- name='customer name %d' % i,
- description='customer description %d' % i
- )
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ ),
)
@@ -102,9 +110,9 @@ def test_core_query_caching(n):
conn.execution_options(compiled_cache=cache).execute(
ins,
dict(
- name='customer name %d' % i,
- description='customer description %d' % i
- )
+ name="customer name %d" % i,
+ description="customer description %d" % i,
+ ),
)
@@ -123,20 +131,22 @@ def test_dbapi_raw_w_pool(n):
def _test_dbapi_raw(n, connect):
- compiled = Customer.__table__.insert().values(
- name=bindparam('name'),
- description=bindparam('description')).\
- compile(dialect=engine.dialect)
+ compiled = (
+ Customer.__table__.insert()
+ .values(name=bindparam("name"), description=bindparam("description"))
+ .compile(dialect=engine.dialect)
+ )
if compiled.positional:
args = (
- ('customer name %d' % i, 'customer description %d' % i)
- for i in range(n))
+ ("customer name %d" % i, "customer description %d" % i)
+ for i in range(n)
+ )
else:
args = (
dict(
- name='customer name %d' % i,
- description='customer description %d' % i
+ name="customer name %d" % i,
+ description="customer description %d" % i,
)
for i in range(n)
)
@@ -162,5 +172,5 @@ def _test_dbapi_raw(n, connect):
conn.close()
-if __name__ == '__main__':
+if __name__ == "__main__":
Profiler.main()