From 26e138cb47dccc859ff219f108ce9b7d96cbcbcd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 3 Jun 2010 18:21:05 +0200 Subject: odb: fixed streamed decompression reader ( specific tests would still be missing ) and added performance tests which are extremely promising --- test/git/performance/test_streams.py | 91 ++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 test/git/performance/test_streams.py (limited to 'test/git/performance/test_streams.py') diff --git a/test/git/performance/test_streams.py b/test/git/performance/test_streams.py new file mode 100644 index 00000000..15924c08 --- /dev/null +++ b/test/git/performance/test_streams.py @@ -0,0 +1,91 @@ +"""Performance data streaming performance""" + +from test.testlib import * +from git.odb.db import * + +from array import array +from cStringIO import StringIO +from time import time +import os +import sys +import stat +import random + + +from lib import ( + TestBigRepoReadOnly + ) + + + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return actual_size*4, StringIO(a.tostring()) + + +class TestObjDBPerformance(TestBigRepoReadOnly): + + large_data_size_bytes = 1000*1000*10 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_bare_rw_repo + def test_large_data_streaming(self, rwrepo): + ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.to_object('blob', size, stream) + elapsed = time() - st + assert ldb.has_object(sha) + fsize_kib = os.path.getsize(ldb.readable_db_object_path(sha)) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed, size_kib / elapsed) + + # reading all at once + st = time() + type, size, shastream = ldb.object(sha) + shadata = shastream.read() + elapsed = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed, size_kib / elapsed) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + type, size, shastream = ldb.object(sha) + while True: + data = shastream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed, size_kib / elapsed) + # END for each randomization factor -- cgit v1.2.1 From 4b4a514e51fbc7dc6ddcb27c188159d57b5d1fa9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 3 Jun 2010 19:04:18 +0200 Subject: Added performance comparison to cgit ... and yes, git-python is faster :) --- test/git/performance/test_streams.py | 67 ++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) (limited to 'test/git/performance/test_streams.py') diff --git a/test/git/performance/test_streams.py b/test/git/performance/test_streams.py index 15924c08..6c2834b3 100644 --- a/test/git/performance/test_streams.py +++ b/test/git/performance/test_streams.py @@ -10,6 +10,7 @@ import os import sys import stat import random +import subprocess from lib import ( @@ -51,23 +52,24 @@ class TestObjDBPerformance(TestBigRepoReadOnly): # writing - due to the compression it will seem faster than it is st = time() sha = ldb.to_object('blob', size, stream) - elapsed = time() - st + elapsed_add = time() - st assert ldb.has_object(sha) - fsize_kib = os.path.getsize(ldb.readable_db_object_path(sha)) / 1000 + db_file = ldb.readable_db_object_path(sha) + fsize_kib = os.path.getsize(db_file) / 1000 size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed, size_kib / elapsed) + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) # reading all at once st = time() type, size, shastream = ldb.object(sha) shadata = shastream.read() - elapsed = time() - st + elapsed_readall = time() - st stream.seek(0) assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed, size_kib / elapsed) + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) # reading in chunks of 1 MiB @@ -81,11 +83,62 @@ class TestObjDBPerformance(TestBigRepoReadOnly): if len(data) < cs: break # END read in chunks - elapsed = time() - st + elapsed_readchunks = time() - st stream.seek(0) assert ''.join(chunks) == stream.getvalue() cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed, size_kib / elapsed) + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so git has something to do + os.remove(db_file) + + # VS. CGIT + ########## + # CGIT ! Can using the cgit programs be faster ? + proc = rwrepo.git.hash_object('-w', '--stdin', as_process=True, istream=subprocess.PIPE) + + # write file - pump everything in at once to be a fast as possible + data = stream.getvalue() # cache it + st = time() + proc.stdin.write(data) + proc.stdin.close() + gitsha = proc.stdout.read().strip() + proc.wait() + gelapsed_add = time() - st + del(data) + assert gitsha == sha # we do it the same way, right ? + + # as its the same sha, we reuse our path + fsize_kib = os.path.getsize(db_file) / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to using git-hash-object in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, gelapsed_add, size_kib / gelapsed_add) + + # compare ... + print >> sys.stderr, "Git-Python is %f %% faster than git when adding big %s files" % (100.0 - (elapsed_add / gelapsed_add) * 100, desc) + + + # read all + st = time() + s, t, size, data = rwrepo.git.get_object_data(gitsha) + gelapsed_readall = time() - st + print >> sys.stderr, "Read %i KiB of %s data at once using git-cat-file in %f s ( %f Read KiB / s)" % (size_kib, desc, gelapsed_readall, size_kib / gelapsed_readall) + + # compare + print >> sys.stderr, "Git-Python is %f %% faster than git when reading big %sfiles" % (100.0 - (elapsed_readall / gelapsed_readall) * 100, desc) + + + # read chunks + st = time() + s, t, size, stream = rwrepo.git.stream_object_data(gitsha) + while True: + data = stream.read(cs) + if len(data) < cs: + break + # END read stream + gelapsed_readchunks = time() - st + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from git-cat-file in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, gelapsed_readchunks, size_kib / gelapsed_readchunks) + + # compare + print >> sys.stderr, "Git-Python is %f %% faster than git when reading big %s files in chunks" % (100.0 - (elapsed_readchunks / gelapsed_readchunks) * 100, desc) # END for each randomization factor -- cgit v1.2.1 From 1e2b46138ba58033738a24dadccc265748fce2ca Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 3 Jun 2010 23:20:34 +0200 Subject: commit.create_from_tree now uses pure python implementation, fixed message parsing which truncated newlines although it was ilegitimate. Its up to the reader to truncate therse, nowhere in the git code I could find anyone adding newlines to commits where it is written Added performance tests for serialization, it does about 5k commits per second if writing to tmpfs --- test/git/performance/test_streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/git/performance/test_streams.py') diff --git a/test/git/performance/test_streams.py b/test/git/performance/test_streams.py index 6c2834b3..d31bee14 100644 --- a/test/git/performance/test_streams.py +++ b/test/git/performance/test_streams.py @@ -14,7 +14,7 @@ import subprocess from lib import ( - TestBigRepoReadOnly + TestBigRepoR ) @@ -32,7 +32,7 @@ def make_memory_file(size_in_bytes, randomize=False): return actual_size*4, StringIO(a.tostring()) -class TestObjDBPerformance(TestBigRepoReadOnly): +class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB -- cgit v1.2.1 From a1e80445ad5cb6da4c0070d7cb8af89da3b0803b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 4 Jun 2010 14:41:15 +0200 Subject: initial version of new odb design to facilitate a channel based multi-threading implementation of all odb functions --- test/git/performance/test_streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'test/git/performance/test_streams.py') diff --git a/test/git/performance/test_streams.py b/test/git/performance/test_streams.py index d31bee14..30fd8048 100644 --- a/test/git/performance/test_streams.py +++ b/test/git/performance/test_streams.py @@ -51,7 +51,7 @@ class TestObjDBPerformance(TestBigRepoR): # writing - due to the compression it will seem faster than it is st = time() - sha = ldb.to_object('blob', size, stream) + sha = ldb.store('blob', size, stream) elapsed_add = time() - st assert ldb.has_object(sha) db_file = ldb.readable_db_object_path(sha) @@ -63,7 +63,7 @@ class TestObjDBPerformance(TestBigRepoR): # reading all at once st = time() - type, size, shastream = ldb.object(sha) + type, size, shastream = ldbstreamsha) shadata = shastream.read() elapsed_readall = time() - st @@ -76,7 +76,7 @@ class TestObjDBPerformance(TestBigRepoR): cs = 512*1000 chunks = list() st = time() - type, size, shastream = ldb.object(sha) + type, size, shastream = ldbstreamsha) while True: data = shastream.read(cs) chunks.append(data) -- cgit v1.2.1 From e746f96bcc29238b79118123028ca170adc4ff0f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 4 Jun 2010 17:22:08 +0200 Subject: Fixed implementation after design change to deal with it - all tests run, but next there will have to be more through testing --- test/git/performance/test_streams.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'test/git/performance/test_streams.py') diff --git a/test/git/performance/test_streams.py b/test/git/performance/test_streams.py index 30fd8048..01ec9fc4 100644 --- a/test/git/performance/test_streams.py +++ b/test/git/performance/test_streams.py @@ -1,7 +1,7 @@ """Performance data streaming performance""" from test.testlib import * -from git.odb.db import * +from git.odb import * from array import array from cStringIO import StringIO @@ -51,7 +51,7 @@ class TestObjDBPerformance(TestBigRepoR): # writing - due to the compression it will seem faster than it is st = time() - sha = ldb.store('blob', size, stream) + sha = ldb.store(IStream('blob', size, stream)).sha elapsed_add = time() - st assert ldb.has_object(sha) db_file = ldb.readable_db_object_path(sha) @@ -63,8 +63,8 @@ class TestObjDBPerformance(TestBigRepoR): # reading all at once st = time() - type, size, shastream = ldbstreamsha) - shadata = shastream.read() + ostream = ldb.stream(sha) + shadata = ostream.read() elapsed_readall = time() - st stream.seek(0) @@ -76,9 +76,9 @@ class TestObjDBPerformance(TestBigRepoR): cs = 512*1000 chunks = list() st = time() - type, size, shastream = ldbstreamsha) + ostream = ldb.stream(sha) while True: - data = shastream.read(cs) + data = ostream.read(cs) chunks.append(data) if len(data) < cs: break -- cgit v1.2.1