diff options
author | Craig Northway <craig.northway@gmail.com> | 2014-07-25 11:26:48 +1000 |
---|---|---|
committer | Craig Northway <craig.northway@gmail.com> | 2014-07-25 19:19:22 +1000 |
commit | a23d0d8617ba3119069e610fc7b0850a17322726 (patch) | |
tree | d8bc5213dd7e7c7f0befdf65afecb13d5435f873 /git/pack.py | |
parent | 75194159abce545bfa38c3172efb42da9b0017dc (diff) | |
download | gitpython-a23d0d8617ba3119069e610fc7b0850a17322726.tar.gz |
Autopep8 fixes with maximum line length 120
Diffstat (limited to 'git/pack.py')
-rw-r--r-- | git/pack.py | 497 |
1 files changed, 248 insertions, 249 deletions
diff --git a/git/pack.py b/git/pack.py index 09a2a1e3..7b675b10 100644 --- a/git/pack.py +++ b/git/pack.py @@ -4,31 +4,31 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from git.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) + BadObject, + UnsupportedOperation, + ParseError +) from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, +) from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size +) try: from _perf import PackIndexFile_sha_to_index @@ -44,19 +44,19 @@ from base import ( # Amazing ! ODeltaStream, ODeltaPackInfo, ODeltaPackStream, - ) +) from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer +) from struct import ( - pack, - unpack, - ) + pack, + unpack, +) from binascii import crc32 @@ -68,10 +68,8 @@ import sys __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') - - -#{ Utilities +#{ Utilities def pack_object_at(cursor, offset, as_stream): """ @@ -87,7 +85,7 @@ def pack_object_at(cursor, offset, as_stream): type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None - + # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset @@ -104,14 +102,14 @@ def pack_object_at(cursor, offset, as_stream): total_rela_offset = i # REF DELTA elif type_id == REF_DELTA: - total_rela_offset = data_rela_offset+20 + total_rela_offset = data_rela_offset + 20 delta_info = data[data_rela_offset:total_rela_offset] # BASE OBJECT else: # assume its a base object total_rela_offset = data_rela_offset # END handle type id - + abs_data_offset = offset + total_rela_offset if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) @@ -127,6 +125,7 @@ def pack_object_at(cursor, offset, as_stream): # END handle info # END handle stream + def write_stream_to_pack(read, write, zstream, base_crc=None): """Copy a stream as read from read function, zip it, and write the result. Count the number of written bytes and return it @@ -140,30 +139,30 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): crc = 0 if want_crc: crc = base_crc - #END initialize crc - + # END initialize crc + while True: chunk = read(chunk_size) br += len(chunk) compressed = zstream.compress(chunk) bw += len(compressed) write(compressed) # cannot assume return value - + if want_crc: crc = crc32(compressed, crc) - #END handle crc - + # END handle crc + if len(chunk) != chunk_size: break - #END copy loop - + # END copy loop + compressed = zstream.flush() bw += len(compressed) write(compressed) if want_crc: crc = crc32(compressed, crc) - #END handle crc - + # END handle crc + return (br, bw, crc) @@ -171,83 +170,84 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): class IndexWriter(object): + """Utility to cache index information, allowing to write all information later in one go to the given stream :note: currently only writes v2 indices""" __slots__ = '_objs' - + def __init__(self): self._objs = list() - + def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - + def write(self, pack_sha, write): """Write the index file using the given write method :param pack_sha: binary sha over the whole pack that we index :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) - + sha_writer = FlexibleSha1Writer(write) sha_write = sha_writer.write sha_write(PackIndexFile.index_v2_signature) sha_write(pack(">L", PackIndexFile.index_version_default)) - + # fanout - tmplist = list((0,)*256) # fanout or list with 64 bit offsets + tmplist = list((0,) * 256) # fanout or list with 64 bit offsets for t in self._objs: tmplist[ord(t[0][0])] += 1 - #END prepare fanout + # END prepare fanout for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) - tmplist[i+1] += v - #END write each fanout entry + tmplist[i + 1] += v + # END write each fanout entry sha_write(pack('>L', tmplist[255])) - + # sha1 ordered # save calls, that is push them into c sha_write(''.join(t[0] for t in self._objs)) - + # crc32 for t in self._objs: - sha_write(pack('>L', t[1]&0xffffffff)) - #END for each crc - + sha_write(pack('>L', t[1] & 0xffffffff)) + # END for each crc + tmplist = list() # offset 32 for t in self._objs: ofs = t[2] if ofs > 0x7fffffff: tmplist.append(ofs) - ofs = 0x80000000 + len(tmplist)-1 - #END hande 64 bit offsets - sha_write(pack('>L', ofs&0xffffffff)) - #END for each offset - + ofs = 0x80000000 + len(tmplist) - 1 + # END hande 64 bit offsets + sha_write(pack('>L', ofs & 0xffffffff)) + # END for each offset + # offset 64 for ofs in tmplist: sha_write(pack(">Q", ofs)) - #END for each offset - + # END for each offset + # trailer assert(len(pack_sha) == 20) sha_write(pack_sha) sha = sha_writer.sha(as_hex=False) write(sha) return sha - - + class PackIndexFile(LazyMixin): + """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" - + # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -258,7 +258,7 @@ class PackIndexFile(LazyMixin): def __init__(self, indexpath): super(PackIndexFile, self).__init__() self._indexpath = indexpath - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -271,128 +271,127 @@ class PackIndexFile(LazyMixin): self._cursor = mman.make_cursor(self._indexpath).use_region() # We will assume that the index will always fully fit into memory ! if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): - raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) - #END assert window size + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % ( + self._indexpath, self._cursor.file_size(), mman.window_size())) + # END assert window size else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties - + # CHECK VERSION mmap = self._cursor.map() self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version - + # SETUP FUNCTIONS # setup our functions according to the actual version for fname in ('entry', 'offset', 'sha', 'crc'): setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) # END for each function to initialize - - + # INITIALIZE DATA # byte offset is 8 if version is 2, 0 otherwise self._initialize() # END handle attributes - #{ Access V1 - + def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - + return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, ) + def _offset_v1(self, i): """see ``_offset_v2``""" - return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - + return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0] + def _sha_v1(self, i): """see ``_sha_v2``""" - base = 1024 + (i*24)+4 - return self._cursor.map()[base:base+20] - + base = 1024 + (i * 24) + 4 + return self._cursor.map()[base:base + 20] + def _crc_v1(self, i): """unsupported""" return 0 - + #} END access V1 - + #{ Access V2 def _entry_v2(self, i): """:return: tuple(offset, binsha, crc)""" return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - + def _offset_v2(self, i): """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - + # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset - + return offset - + def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 - return self._cursor.map()[base:base+20] - + return self._cursor.map()[base:base + 20] + def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + #} END access V2 - + #{ Initialization - + def _initialize(self): """initialize base data""" self._fanout_table = self._read_fanout((self._version == 2) * 8) - + if self._version == 2: self._crc_list_offset = self._sha_list_offset + self.size() * 20 self._pack_offset = self._crc_list_offset + self.size() * 4 self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base - + def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" d = self._cursor.map() out = list() append = out.append for i in range(256): - append(unpack_from('>L', d, byte_offset + i*4)[0]) + append(unpack_from('>L', d, byte_offset + i * 4)[0]) # END for each entry return out - + #} END initialization - + #{ Properties def version(self): return self._version - + def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - + def path(self): """:return: path to the packindexfile""" return self._indexpath - + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._cursor.map()[-40:-20] - + def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._cursor.map()[-20:] - + def offsets(self): """:return: sequence of all offsets in the order in which they were written :note: return value can be random accessed, but may be immmutable""" @@ -400,7 +399,7 @@ class PackIndexFile(LazyMixin): # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - + # networkbyteorder to something array likes more if sys.byteorder == 'little': a.byteswap() @@ -408,7 +407,7 @@ class PackIndexFile(LazyMixin): else: return tuple(self.offset(index) for index in xrange(self.size())) # END handle version - + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -418,9 +417,9 @@ class PackIndexFile(LazyMixin): get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: - lo = self._fanout_table[first_byte-1] + lo = self._fanout_table[first_byte - 1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 @@ -434,7 +433,7 @@ class PackIndexFile(LazyMixin): # END handle midpoint # END bisect return None - + def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this @@ -445,18 +444,18 @@ class PackIndexFile(LazyMixin): :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: - lo = self._fanout_table[first_byte-1] + lo = self._fanout_table[first_byte - 1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest + filled_sha = partial_bin_sha + '\0' * (20 - len(partial_bin_sha)) + + # find lowest while lo < hi: mid = (lo + hi) / 2 c = cmp(filled_sha, get_sha(mid)) @@ -470,99 +469,99 @@ class PackIndexFile(LazyMixin): lo = mid + 1 # END handle midpoint # END bisect - + if lo < self.size(): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size(): - next_sha = get_sha(lo+1) + if lo + 1 < self.size(): + next_sha = get_sha(lo + 1) if next_sha and next_sha == cur_sha: raise AmbiguousObjectName(partial_bin_sha) return lo # END if we have a match # END if we found something return None - + if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses def sha_to_index(self, sha): return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - + # END redefine heavy-hitter with c version + #} END properties - - + + class PackFile(LazyMixin): + """A pack is a file written according to the Version 2 for git packs - + As we currently use memory maps, it could be assumed that the maximum size of packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - + :note: at some point, this might be implemented using streams as well, or streams are an alternate path in the case memory maps cannot be created for some reason - one clearly doesn't want to read 10GB at once in that case""" - + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 - + # offset into our data at which the first object starts - first_object_offset = 3*4 # header bytes + first_object_offset = 3 * 4 # header bytes footer_size = 20 # final sha - + def __init__(self, packpath): self._packpath = packpath - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() - + # read the header information type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - + # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" c = self._cursor content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset - + null = NullStream() while cur_offset < content_size: data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset - + stream_copy(ostream.read, null.write, ostream.size, chunk_size) cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - + # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the + # Otherwise return the Stream object directly, its derived from the # info object if as_stream: ostream.stream.seek(0) yield ostream # END until we have read everything - + #{ Pack Information - + def size(self): - """:return: The amount of objects stored in this pack""" + """:return: The amount of objects stored in this pack""" return self._size - + def version(self): """:return: the version of this pack""" return self._version - + def data(self): """ :return: read-only data of this pack. It provides random access and usually @@ -570,18 +569,18 @@ class PackFile(LazyMixin): :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" # can use map as we are starting at offset 0. Otherwise we would have to use buffer() return self._cursor.use_region().map() - + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - + return self._cursor.use_region(self._cursor.file_size() - 20).buffer()[:] + def path(self): """:return: path to the packfile""" return self._packpath #} END pack information - + #{ Pack Specific - + def collect_streams(self, offset): """ :return: list of pack streams which are required to build the object @@ -600,7 +599,7 @@ class PackFile(LazyMixin): offset = ostream.pack_offset - ostream.delta_info else: # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter + # else is either an object, or a ref delta, in the latter # case someone else has to find it break # END handle type @@ -608,23 +607,23 @@ class PackFile(LazyMixin): return out #} END pack specific - + #{ Read-Database like Interface - + def info(self, offset): """Retrieve information about the object at the given file-absolute offset - + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - + def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information - + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - + def stream_iter(self, start_offset=0): """ :return: iterator yielding OPackStream compatible instances, allowing @@ -634,28 +633,29 @@ class PackFile(LazyMixin): :note: Iterating a pack directly is costly as the datastream has to be decompressed to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) - + #} END Read-Database like Interface - - + + class PackEntity(LazyMixin): + """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file - '_pack', # our pack file - '_offset_map' # on demand dict mapping one offset to the next consecutive one - ) - + + __slots__ = ('_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) + IndexFileCls = PackIndexFile PackFileCls = PackFile - + def __init__(self, pack_or_index_path): """Initialize ourselves with the path to the respective pack or index file""" basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - + def _set_cache_(self, attr): # currently this can only be _offset_map # TODO: make this a simple sorted offset array which can be bisected @@ -664,30 +664,30 @@ class PackEntity(LazyMixin): offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" - + offset_map = None if len(offsets_sorted) == 1: - offset_map = { offsets_sorted[0] : last_offset } + offset_map = {offsets_sorted[0]: last_offset} else: iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) iter_offsets_plus_one.next() consecutive = izip(iter_offsets, iter_offsets_plus_one) - + offset_map = dict(consecutive) - + # the last offset is not yet set offset_map[offsets_sorted[-1]] = last_offset # END handle offset amount self._offset_map = offset_map - + def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) if index is None: raise BadObject(sha) return index - + def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha @@ -695,7 +695,7 @@ class PackEntity(LazyMixin): for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index - + def _object(self, sha, as_stream, index=-1): """:return: OInfo or OStream object providing information about the given sha :param index: if not -1, its assumed to be the sha's index in the IndexFile""" @@ -712,45 +712,45 @@ class PackEntity(LazyMixin): packstream = self._pack.stream(offset) return OStream(sha, packstream.type, packstream.size, packstream.stream) # END handle non-deltas - + # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, + # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) + + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) # END handle non-deltas - + # deltas are a little tougher - unpack the first bytes to obtain # the actual target size, as opposed to the size of the delta data streams = self.collect_streams_at_offset(offset) buf = streams[0].read(512) offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - + # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) + return OInfo(sha, streams[-1].type, target_size) # END handle stream - + #{ Read-Database like Interface - + def info(self, sha): """Retrieve information about the object identified by the given sha - + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" return self._object(sha, False) - + def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha - + :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" @@ -759,28 +759,28 @@ class PackEntity(LazyMixin): def info_at_index(self, index): """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, False, index) - + def stream_at_index(self, index): """As ``stream``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, True, index) - + #} END Read-Database like Interface - - #{ Interface + + #{ Interface def pack(self): """:return: the underlying pack file instance""" return self._pack - + def index(self): """:return: the underlying pack index file instance""" return self._index - + def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - + :param use_crc: if True, the index' crc is run over the compressed stream of the object, which is much faster than checking the sha1. It is also more prone to unnoticed corruption or manipulation. @@ -791,7 +791,7 @@ class PackEntity(LazyMixin): just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha - + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -799,12 +799,12 @@ class PackEntity(LazyMixin): if self._index.version() < 2: raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") # END handle index version - + index = self._sha_to_index(sha) offset = self._index.offset(index) next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - + # create the current crc value, on the compressed object data # Read it in chunks, without copying the data crc_update = zlib.crc32 @@ -817,7 +817,7 @@ class PackEntity(LazyMixin): this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - + # crc returns signed 32 bit numbers, the AND op forces it into unsigned # mode ... wow, sneaky, from dulwich. return (this_crc_value & 0xffffffff) == crc_value @@ -826,7 +826,7 @@ class PackEntity(LazyMixin): stream = self._object(sha, as_stream=True) # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) - + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification @@ -837,21 +837,21 @@ class PackEntity(LazyMixin): :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) - + def stream_iter(self): """ :return: iterator over all objects in this pack. The iterator yields OStream instances""" return self._iter_objects(as_stream=True) - + def collect_streams_at_offset(self, offset): """ As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` - + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) - + # try to resolve the last one if needed. It is assumed to be either # a REF delta, or a base object, as OFFSET deltas are resolved by the pack if streams[-1].type_id == REF_DELTA: @@ -864,17 +864,17 @@ class PackEntity(LazyMixin): stream = self._pack.stream(self._index.offset(sindex)) streams.append(stream) else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who # would do that ;) ? We can handle it though stream = self._pack.stream(stream.delta_info) streams.append(stream) # END handle ref delta # END resolve ref streams # END resolve streams - + return streams - + def collect_streams(self, sha): """ As ``PackFile.collect_streams``, but takes a sha instead of an offset. @@ -882,22 +882,21 @@ class PackEntity(LazyMixin): If this is not possible, the stream will be left alone, hence it is adivsed to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. - + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. :raise BadObject:""" return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - + @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, - object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count=None, zlib_compression=zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. The respective index is produced as well if index_write is not Non. - + :param object_iter: iterator yielding odb output objects :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding @@ -915,72 +914,73 @@ class PackEntity(LazyMixin): if not object_count: if not isinstance(object_iter, (tuple, list)): objs = list(object_iter) - #END handle list type + # END handle list type object_count = len(objs) - #END handle object - + # END handle object + pack_writer = FlexibleSha1Writer(pack_write) pwrite = pack_writer.write ofs = 0 # current offset into the pack file index = None wants_index = index_write is not None - + # write header pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) ofs += 12 - + if wants_index: index = IndexWriter() - #END handle index header - + # END handle index header + actual_count = 0 for obj in objs: actual_count += 1 crc = 0 - + # object header hdr = create_pack_object_header(obj.type_id, obj.size) if index_write: crc = crc32(hdr) else: crc = None - #END handle crc + # END handle crc pwrite(hdr) - + # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc=crc) assert(br == obj.size) if wants_index: index.append(obj.binsha, crc, ofs) - #END handle index - + # END handle index + ofs += len(hdr) + bw if actual_count == object_count: break - #END abort once we are done - #END for each object - + # END abort once we are done + # END for each object + if actual_count != object_count: - raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) - #END count assertion - + raise ValueError( + "Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + # END count assertion + # write footer - pack_sha = pack_writer.sha(as_hex = False) + pack_sha = pack_writer.sha(as_hex=False) assert len(pack_sha) == 20 pack_write(pack_sha) ofs += len(pack_sha) # just for completeness ;) - + index_sha = None if wants_index: index_sha = index.write(pack_sha, index_write) - #END handle index - + # END handle index + return pack_sha, index_sha - + @classmethod - def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.Z_BEST_SPEED): """Create a new on-disk entity comprised of a properly named pack file and a properly named and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files @@ -990,18 +990,17 @@ class PackEntity(LazyMixin): index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) index_write = lambda d: os.write(index_fd, d) - + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) os.close(pack_fd) os.close(index_fd) - + fmt = "pack-%s.%s" new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) os.rename(pack_path, new_pack_path) os.rename(index_path, new_index_path) - + return cls(new_pack_path) - - + #} END interface |