diff options
| -rw-r--r-- | lib/git/odb/db.py | 15 | ||||
| -rw-r--r-- | lib/git/odb/stream.py | 29 | ||||
| -rw-r--r-- | test/git/test_odb.py | 55 |
3 files changed, 66 insertions, 33 deletions
diff --git a/lib/git/odb/db.py b/lib/git/odb/db.py index a8de28ec..d970b0bf 100644 --- a/lib/git/odb/db.py +++ b/lib/git/odb/db.py @@ -29,7 +29,8 @@ from utils import ( from fun import ( chunk_size, loose_object_header_info, - write_object + write_object, + stream_copy ) import tempfile @@ -262,7 +263,6 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): def store(self, istream): """note: The sha we produce will be hex by nature""" - assert istream.sha is None, "Direct istream writing not yet implemented" tmp_path = None writer = self.ostream() if writer is None: @@ -273,8 +273,13 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): try: try: - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies except: if tmp_path: os.remove(tmp_path) @@ -285,7 +290,7 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): writer.close() # END assure target stream is closed - sha = writer.sha(as_hex=True) + sha = istream.sha or writer.sha(as_hex=True) if tmp_path: obj_path = self.db_path(self.object_path(sha)) diff --git a/lib/git/odb/stream.py b/lib/git/odb/stream.py index 654bcbf6..da97cf5b 100644 --- a/lib/git/odb/stream.py +++ b/lib/git/odb/stream.py @@ -69,15 +69,6 @@ class OStream(OInfo): def __init__(self, *args, **kwargs): tuple.__init__(self) - #{ Interface - - def is_compressed(self): - """:return: True if reads of this stream yield zlib compressed data. Default False - :note: this does not imply anything about the actual internal storage. - Hence the data could be uncompressed, but read compressed, or vice versa""" - return False - - #} END interface #{ Stream Reader Interface @@ -97,11 +88,11 @@ class IStream(list): The only method your content stream must support is 'read'""" __slots__ = tuple() - def __new__(cls, type, size, stream, sha=None, compressed=False): - return list.__new__(cls, (sha, type, size, stream, compressed, None)) + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) - def __init__(self, type, size, stream, sha=None, compressed=None): - list.__init__(self, (sha, type, size, stream, compressed, None)) + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) #{ Interface @@ -117,11 +108,11 @@ class IStream(list): def _error(self): """:return: the error that occurred when processing the stream, or None""" - return self[5] + return self[4] def _set_error(self, exc): """Set this input stream to the given exc, may be None to reset the error""" - self[5] = exc + self[4] = exc error = property(_error, _set_error) @@ -172,13 +163,6 @@ class IStream(list): stream = property(_stream, _set_stream) #} END odb info interface - - #{ OStream interface - - def is_compressed(self): - return self[4] - - #} END OStream interface class InvalidOInfo(tuple): @@ -397,6 +381,7 @@ class DecompressMemMapReader(object): class Sha1Writer(object): """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" + __slots__ = "sha1" def __init__(self): self.sha1 = make_sha("") diff --git a/test/git/test_odb.py b/test/git/test_odb.py index 2f8ebd17..5c8268cd 100644 --- a/test/git/test_odb.py +++ b/test/git/test_odb.py @@ -31,7 +31,8 @@ class DummyStream(object): def _assert(self): assert self.was_read - + + class DeriveTest(OStream): def __init__(self, sha, type, size, stream, *args, **kwargs): self.myarg = kwargs.pop('myarg') @@ -41,6 +42,27 @@ class DeriveTest(OStream): assert self.args assert self.myarg + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(1) # fastest + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + #} END stream utilitiess @@ -68,15 +90,11 @@ class TestStream(TestBase): ostream.read(20) assert stream.bytes == 20 - # defaults false - assert not ostream.is_compressed() - # derive with own args DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() # test istream istream = IStream(Blob.type, s, stream) - assert not istream.is_compressed() assert istream.sha == None istream.sha = sha assert istream.sha == sha @@ -94,6 +112,10 @@ class TestStream(TestBase): istream.stream = None assert istream.stream is None + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): """Make stream tests - the orig_stream is seekable, allowing it to be rewound and reused @@ -218,13 +240,14 @@ class TestDB(TestBase): """General tests to verify object writing, compatible to ObjectDBW :note: requires write access to the database""" # start in 'dry-run' mode, using a simple sha1 writer - ostreams = (Sha1Writer, None) + ostreams = (ZippedStoreShaWriter, None) for ostreamcls in ostreams: for data in self.all_data: dry_run = ostreamcls is not None ostream = None if ostreamcls is not None: ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) # END create ostream prev_ostream = db.set_ostream(ostream) @@ -252,6 +275,26 @@ class TestDB(TestBase): else: self.failUnlessRaises(BadObject, db.info, sha) self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.sha is not None + prev_sha = istream.sha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.sha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip + assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode |