From 4a534eba97db3c2cfb2926368756fd633d25c056 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 20 Oct 2009 12:24:47 +0200 Subject: Added frame for index implementation and testing --- lib/git/index.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/git/index.py (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py new file mode 100644 index 00000000..1c531712 --- /dev/null +++ b/lib/git/index.py @@ -0,0 +1,17 @@ +# index.py +# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php +""" +Module containing Index implementation, allowing to perform all kinds of index +manipulations such as querying and merging. +""" + +class Index(object): + """ + Implements an Index that can be manipulated using a native implementation in + order to safe git command function calls wherever possible. + + It provides custom merging facilities and to create custom commits. + """ -- cgit v1.2.1 From 50a9920b1bd9e6e8cf452c774c499b0b9014ccef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 20 Oct 2009 17:04:23 +0200 Subject: Added initial version of the index reading from file - IndexEntry interface is to be improved though, writing needs to be implemented as well --- lib/git/index.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 1c531712..1e67d6d1 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -7,11 +7,109 @@ Module containing Index implementation, allowing to perform all kinds of index manipulations such as querying and merging. """ +import struct +import binascii +import mmap + +class IndexEntry(tuple): + """ + Allows convenient access to IndexEntry data without completely unpacking it. + + Attributes usully accessed often are cached in the tuple whereas others are + unpacked on demand. + """ class Index(object): """ Implements an Index that can be manipulated using a native implementation in - order to safe git command function calls wherever possible. + order to save git command function calls wherever possible. It provides custom merging facilities and to create custom commits. """ + __slots__ = ( "version", "entries" ) + + def __init__(self, stream = None): + """ + Initialize this Index instance, optionally from the given ``stream`` + + Note + Reading is based on the dulwich project. + """ + self.entries = dict() + self.version = -1 + if stream is not None: + self._read_from_stream(stream) + + def _read_entry(self, stream): + """Return: One entry of the given stream""" + beginoffset = stream.tell() + ctime = struct.unpack(">8s", stream.read(8))[0] + mtime = struct.unpack(">8s", stream.read(8))[0] + (dev, ino, mode, uid, gid, size, sha, flags) = \ + struct.unpack(">LLLLLL20sH", stream.read(20 + 4 * 6 + 2)) + path_size = flags & 0x0fff + path = stream.read(path_size) + + real_size = ((stream.tell() - beginoffset + 8) & ~7) + data = stream.read((beginoffset + real_size) - stream.tell()) + return IndexEntry((path, ctime, mtime, dev, ino, mode, uid, gid, size, + binascii.hexlify(sha), path_size)) + + + def _read_header(self, stream): + """Return tuple(version_long, num_entries) from the given stream""" + type_id = stream.read(4) + if type_id != "DIRC": + raise AssertionError("Invalid index file header: %r" % type_id) + version, num_entries = struct.unpack(">LL", stream.read(4 * 2)) + assert version in (1, 2) + return version, num_entries + + def _read_from_stream(self, stream): + """ + Initialize this instance with index values read from the given stream + """ + self.version, num_entries = self._read_header(stream) + self.entries = dict() + count = 0 + while count < num_entries: + entry = self._read_entry(stream) + self.entries[entry[0]] = entry[1:] + count += 1 + # END for each entry + + @classmethod + def from_file(cls, file_path): + """ + Returns + Index instance as recreated from the given stream. + + ``file_pa `` + File path pointing to git index file + """ + fp = open(file_path, "r") + + # try memory map for speed + stream = fp + try: + stream = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) + except Exception: + pass + # END memory mapping + + try: + return cls(stream) + finally: + fp.close() + + def write(self, stream): + """ + Write the current state to the given stream + + ``stream`` + File-like object + + Returns + self + """ + raise NotImplementedError( "TODO" ) -- cgit v1.2.1 From 56823868efddd3bdbc0b624cdc79adc3a2e94a75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 20 Oct 2009 21:32:00 +0200 Subject: Improved tuple access of EntryIndex class including test, stage and type access still needs to be decoded though --- lib/git/index.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 1e67d6d1..1ce4183b 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -10,6 +10,7 @@ manipulations such as querying and merging. import struct import binascii import mmap +import objects class IndexEntry(tuple): """ @@ -17,7 +18,61 @@ class IndexEntry(tuple): Attributes usully accessed often are cached in the tuple whereas others are unpacked on demand. + + See the properties for a mapping between names and tuple indices. """ + @property + def path(self): + return self[0] + + @property + def ctime(self): + """ + Returns + Tuple(int_time_seconds_since_epoch, int_nano_seconds) of the + file's creation time + """ + return struct.unpack(">LL", self[1]) + + @property + def mtime(self): + """ + See ctime property, but returns modification time + """ + return struct.unpack(">LL", self[2]) + + @property + def dev(self): + return self[3] + + @property + def inode(self): + return self[4] + + @property + def mode(self): + return self[5] + + @property + def uid(self): + return self[6] + + @property + def gid(self): + return self[7] + + @property + def data_size(self): + return self[8] + + @property + def sha(self): + return self[9] + + @property + def path_size(self): + return self[10] + class Index(object): """ -- cgit v1.2.1 From 152bab7eb64e249122fefab0d5531db1e065f539 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 20 Oct 2009 22:05:51 +0200 Subject: improved IndexEntry type and added test for parsing of the stage --- lib/git/index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 1ce4183b..ad581ad4 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -62,7 +62,7 @@ class IndexEntry(tuple): return self[7] @property - def data_size(self): + def size(self): return self[8] @property @@ -70,7 +70,7 @@ class IndexEntry(tuple): return self[9] @property - def path_size(self): + def stage(self): return self[10] @@ -108,7 +108,7 @@ class Index(object): real_size = ((stream.tell() - beginoffset + 8) & ~7) data = stream.read((beginoffset + real_size) - stream.tell()) return IndexEntry((path, ctime, mtime, dev, ino, mode, uid, gid, size, - binascii.hexlify(sha), path_size)) + binascii.hexlify(sha), flags >> 12)) def _read_header(self, stream): @@ -129,7 +129,7 @@ class Index(object): count = 0 while count < num_entries: entry = self._read_entry(stream) - self.entries[entry[0]] = entry[1:] + self.entries[(entry.path,entry.stage)] = entry count += 1 # END for each entry -- cgit v1.2.1 From b9d6494f1075e5370a20e406c3edb102fca12854 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Oct 2009 13:34:43 +0200 Subject: index writing added including simple test, improved docs of IndexEntry --- lib/git/index.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 127 insertions(+), 25 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index ad581ad4..c137d4da 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -21,10 +21,6 @@ class IndexEntry(tuple): See the properties for a mapping between names and tuple indices. """ - @property - def path(self): - return self[0] - @property def ctime(self): """ @@ -32,45 +28,82 @@ class IndexEntry(tuple): Tuple(int_time_seconds_since_epoch, int_nano_seconds) of the file's creation time """ - return struct.unpack(">LL", self[1]) + return struct.unpack(">LL", self[0]) @property def mtime(self): """ See ctime property, but returns modification time """ - return struct.unpack(">LL", self[2]) + return struct.unpack(">LL", self[1]) @property def dev(self): - return self[3] + """ + Device ID + """ + return self[2] @property def inode(self): - return self[4] + """ + Inode ID + """ + return self[3] @property def mode(self): - return self[5] + """ + File Mode, compatible to stat module constants + """ + return self[4] @property def uid(self): - return self[6] + """ + User ID + """ + return self[5] @property def gid(self): - return self[7] + """ + Group ID + """ + return self[6] @property def size(self): - return self[8] + """ + Uncompressed size of the blob + + Note + Will be 0 if the stage is not 0 ( hence it is an unmerged entry ) + """ + return self[7] @property def sha(self): - return self[9] + """ + hex sha of the blob + """ + return self[8] @property def stage(self): + """ + Stage of the entry, either: + 0 = default stage + 1 = stage before a merge or common ancestor entry in case of a 3 way merge + 2 = stage of entries from the 'left' side of the merge + 3 = stage of entries from the right side of the merge + Note: + For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html + """ + return self[9] + + @property + def path(self): return self[10] @@ -80,22 +113,25 @@ class Index(object): order to save git command function calls wherever possible. It provides custom merging facilities and to create custom commits. + + ``Entries`` + The index contains an entries dict whose keys are tuples of """ - __slots__ = ( "version", "entries" ) + __slots__ = ( "version", "entries", "_extension_data" ) + _VERSION = 2 # latest version we support def __init__(self, stream = None): """ Initialize this Index instance, optionally from the given ``stream`` - - Note - Reading is based on the dulwich project. """ self.entries = dict() - self.version = -1 + self.version = self._VERSION + self._extension_data = '' if stream is not None: self._read_from_stream(stream) - def _read_entry(self, stream): + @classmethod + def _read_entry(cls, stream): """Return: One entry of the given stream""" beginoffset = stream.tell() ctime = struct.unpack(">8s", stream.read(8))[0] @@ -107,11 +143,11 @@ class Index(object): real_size = ((stream.tell() - beginoffset + 8) & ~7) data = stream.read((beginoffset + real_size) - stream.tell()) - return IndexEntry((path, ctime, mtime, dev, ino, mode, uid, gid, size, - binascii.hexlify(sha), flags >> 12)) + return IndexEntry((ctime, mtime, dev, ino, mode, uid, gid, size, + binascii.hexlify(sha), flags >> 12, path)) - - def _read_header(self, stream): + @classmethod + def _read_header(cls, stream): """Return tuple(version_long, num_entries) from the given stream""" type_id = stream.read(4) if type_id != "DIRC": @@ -123,15 +159,20 @@ class Index(object): def _read_from_stream(self, stream): """ Initialize this instance with index values read from the given stream + + Note + We explicitly do not clear the entries dict here to allow for reading + multiple chunks from multiple streams into the same Index instance """ self.version, num_entries = self._read_header(stream) - self.entries = dict() count = 0 while count < num_entries: entry = self._read_entry(stream) self.entries[(entry.path,entry.stage)] = entry count += 1 # END for each entry + # this data chunk is the footer of the index, don't yet know what it is for + self._extension_data = stream.read(~0) @classmethod def from_file(cls, file_path): @@ -141,6 +182,9 @@ class Index(object): ``file_pa `` File path pointing to git index file + + Note + Reading is based on the dulwich project. """ fp = open(file_path, "r") @@ -157,6 +201,49 @@ class Index(object): finally: fp.close() + + @classmethod + def to_file(cls, index, file_path): + """ + Write the index data to the given file path. + + ``index`` + Index you wish to write. + + ``file_path`` + Path at which to write the index data. Please note that missing directories + will lead to an exception to be thrown. + + Raise + IOError if the file could not be written + """ + fp = open(file_path, "w") + try: + return index.write(fp) + finally: + fp.close() + # END exception handling + + + @classmethod + def _write_cache_entry(cls, stream, entry): + """ + Write an IndexEntry to a stream + """ + beginoffset = stream.tell() + stream.write(entry[0]) # ctime + stream.write(entry[1]) # mtime + path = entry[10] + plen = len(path) & 0x0fff # path length + assert plen == len(path), "Path %s too long to fit into index" % entry[10] + flags = plen | (entry[9] << 12)# stage and path length are 2 byte flags + stream.write(struct.pack(">LLLLLL20sH", entry[2], entry[3], entry[4], + entry[5], entry[6], entry[7], binascii.unhexlify(entry[8]), flags)) + stream.write(path) + real_size = ((stream.tell() - beginoffset + 8) & ~7) + stream.write("\0" * ((beginoffset + real_size) - stream.tell())) + + def write(self, stream): """ Write the current state to the given stream @@ -166,5 +253,20 @@ class Index(object): Returns self + + Note + Index writing based on the dulwich implementation """ - raise NotImplementedError( "TODO" ) + # header + stream.write("DIRC") + stream.write(struct.pack(">LL", self.version, len(self.entries))) + + # body + entries_sorted = self.entries.values() + entries_sorted.sort(key=lambda e: (e[10], e[9])) # use path/stage as sort key + for entry in entries_sorted: + self._write_cache_entry(stream, entry) + # END for each entry + # write extension_data which we currently cannot interprete + stream.write(self._extension_data) + -- cgit v1.2.1 From babf5765da3e328cc1060cb9b37fbdeb6fd58350 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Oct 2009 16:52:25 +0200 Subject: Initial version of merge including tests for one-way, two-way and tree-way merge --- lib/git/index.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 3 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index c137d4da..6b51c5c7 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -11,6 +11,8 @@ import struct import binascii import mmap import objects +import tempfile +import os class IndexEntry(tuple): """ @@ -115,7 +117,8 @@ class Index(object): It provides custom merging facilities and to create custom commits. ``Entries`` - The index contains an entries dict whose keys are tuples of + The index contains an entries dict whose keys are tuples of type IndexEntry + to facilitate access. """ __slots__ = ( "version", "entries", "_extension_data" ) _VERSION = 2 # latest version we support @@ -243,7 +246,6 @@ class Index(object): real_size = ((stream.tell() - beginoffset + 8) & ~7) stream.write("\0" * ((beginoffset + real_size) - stream.tell())) - def write(self, stream): """ Write the current state to the given stream @@ -269,4 +271,77 @@ class Index(object): # END for each entry # write extension_data which we currently cannot interprete stream.write(self._extension_data) - + + + @classmethod + def from_tree(cls, repo, *treeish, **kwargs): + """ + Merge the given treeish revisions into a new index which is returned. + The original index will remain unaltered + + ``repo`` + The repository treeish are located in. + + ``*treeish`` + One, two or three Tree Objects or Commits. The result changes according to the + amoutn of trees. + If 1 Tree is given, it will just be read into a new index + If 2 Trees are given, they will be merged into a new index using a + two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' + one. + If 3 Trees are given, a 3-way merge will be performed with the first tree + being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree, + tree 3 is the 'other' one + + ``**kwargs`` + Additional arguments passed to git-read-tree + + Note: + In the three-way merge case, --aggressive will be specified to automatically + resolve more cases in a commonly correct manner. Specify trivial=True as kwarg + to override that. + """ + if len(treeish) == 0 or len(treeish) > 3: + raise ValueError("Please specify between 1 and 3 treeish, got %i" % len(treeish)) + + arg_list = list() + # ignore that working tree and index possibly are out of date + if len(treeish)>1: + # drop unmerged entries when reading our index and merging + arg_list.append("--reset") + # handle non-trivial cases the way a real merge does + arg_list.append("--aggressive") + # END merge handling + + # tmp file created in git home directory to be sure renaming + # works - /tmp/ dirs could be on another device + tmp_index = tempfile.mktemp('','',repo.path) + arg_list.append("--index-output=%s" % tmp_index) + arg_list.extend(treeish) + + # move current index out of the way - otherwise the merge may fail + # as it considers existing entries. moving it essentially clears the index. + # Unfortunately there is no 'soft' way to do it + cur_index = os.path.join(repo.path, 'index') + moved_index = os.path.join(repo.path, 'index_moved'+tempfile.mktemp('','','')) + try: + os.rename(cur_index, moved_index) + repo.git.read_tree(*arg_list, **kwargs) + index = cls.from_file(tmp_index) + finally: + # put back the original index first ! + if os.path.exists(moved_index): + os.rename(moved_index, cur_index) + if os.path.exists(tmp_index): + os.remove(tmp_index) + # END index merge handling + + return index + + def write_tree(self, stream): + """ + Writes the + """ + raise NotImplementedError("TODO") + + -- cgit v1.2.1 From d97afa24ad1ae453002357e5023f3a116f76fb17 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Oct 2009 18:40:35 +0200 Subject: Improved testing of index against trees, tests succeed with next commit --- lib/git/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 6b51c5c7..1042d7b8 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -288,7 +288,7 @@ class Index(object): If 1 Tree is given, it will just be read into a new index If 2 Trees are given, they will be merged into a new index using a two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' - one. + one. It behaves like a fast-forward. If 3 Trees are given, a 3-way merge will be performed with the first tree being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree, tree 3 is the 'other' one -- cgit v1.2.1 From 2e68d907022c84392597e05afc22d9fe06bf0927 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Oct 2009 18:45:41 +0200 Subject: tree.traverse: Added prune functionality - previously the predciate did both, pruning and preventing to return items --- lib/git/index.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 1042d7b8..7481f4ce 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -284,7 +284,7 @@ class Index(object): ``*treeish`` One, two or three Tree Objects or Commits. The result changes according to the - amoutn of trees. + amount of trees. If 1 Tree is given, it will just be read into a new index If 2 Trees are given, they will be merged into a new index using a two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' @@ -300,6 +300,10 @@ class Index(object): In the three-way merge case, --aggressive will be specified to automatically resolve more cases in a commonly correct manner. Specify trivial=True as kwarg to override that. + + As the underlying git-read-tree command takes into account the current index, + it will be temporarily moved out of the way to assure there are no unsuspected + interferences. """ if len(treeish) == 0 or len(treeish) > 3: raise ValueError("Please specify between 1 and 3 treeish, got %i" % len(treeish)) -- cgit v1.2.1 From 6662422ba52753f8b10bc053aba82bac3f2e1b9c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Oct 2009 21:25:52 +0200 Subject: index.iter_blobs method added including tests ( which have been improved generally for more coverage ) --- lib/git/index.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 8 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 7481f4ce..e5b2d009 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -13,6 +13,8 @@ import mmap import objects import tempfile import os +import stat +from git.objects import Blob class IndexEntry(tuple): """ @@ -114,19 +116,24 @@ class Index(object): Implements an Index that can be manipulated using a native implementation in order to save git command function calls wherever possible. - It provides custom merging facilities and to create custom commits. + It provides custom merging facilities allowing to merge without actually changing + your index or your working tree. This way you can perform own test-merges based + on the index only without having to deal with the working copy. This is useful + in case of partial working trees. ``Entries`` The index contains an entries dict whose keys are tuples of type IndexEntry to facilitate access. """ - __slots__ = ( "version", "entries", "_extension_data" ) + __slots__ = ( "repo", "version", "entries", "_extension_data" ) _VERSION = 2 # latest version we support + S_IFGITLINK = 0160000 - def __init__(self, stream = None): + def __init__(self, repo, stream = None): """ Initialize this Index instance, optionally from the given ``stream`` """ + self.repo = repo self.entries = dict() self.version = self._VERSION self._extension_data = '' @@ -178,11 +185,14 @@ class Index(object): self._extension_data = stream.read(~0) @classmethod - def from_file(cls, file_path): + def from_file(cls, repo, file_path): """ Returns Index instance as recreated from the given stream. - + + ``repo`` + Repository the index is related to + ``file_pa `` File path pointing to git index file @@ -200,7 +210,7 @@ class Index(object): # END memory mapping try: - return cls(stream) + return cls(repo, stream) finally: fp.close() @@ -331,7 +341,7 @@ class Index(object): try: os.rename(cur_index, moved_index) repo.git.read_tree(*arg_list, **kwargs) - index = cls.from_file(tmp_index) + index = cls.from_file(repo, tmp_index) finally: # put back the original index first ! if os.path.exists(moved_index): @@ -341,7 +351,41 @@ class Index(object): # END index merge handling return index - + + @classmethod + def _index_mode_to_tree_index_mode(cls, index_mode): + """Cleanup a index_mode value. + This will return a index_mode that can be stored in a tree object. + ``index_mode`` + Index_mode to clean up. + """ + if stat.S_ISLNK(index_mode): + return stat.S_IFLNK + elif stat.S_ISDIR(index_mode): + return stat.S_IFDIR + elif stat.S_IFMT(index_mode) == cls.S_IFGITLINK: + return cls.S_IFGITLINK + ret = stat.S_IFREG | 0644 + ret |= (index_mode & 0111) + return ret + + def iter_blobs(self, predicate = lambda t: True): + """ + Returns + Iterator yielding tuples of Blob objects and stages, tuple(stage, Blob) + + ``predicate`` + Function(t) returning True if tuple(stage, Blob) should be yielded by the + iterator + """ + for entry in self.entries.itervalues(): + mode = self._index_mode_to_tree_index_mode(entry.mode) + blob = Blob(self.repo, entry.sha, mode, entry.path) + output = (entry.stage, blob) + if predicate(output): + yield output + # END for each entry + def write_tree(self, stream): """ Writes the -- cgit v1.2.1 From 7b50af0a20bcc7280940ce07593007d17c5acabd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Oct 2009 23:11:40 +0200 Subject: index: Added write_tree method including test --- lib/git/index.py | 53 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index e5b2d009..5bf654fe 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -14,7 +14,24 @@ import objects import tempfile import os import stat -from git.objects import Blob +from git.objects import Blob, Tree + +class _TemporaryFileSwap(object): + """ + Utility class moving a file to a temporary location within the same directory + and moving it back on to where on object deletion. + """ + __slots__ = ("file_path", "tmp_file_path") + + def __init__(self, file_path): + self.file_path = file_path + self.tmp_file_path = self.file_path + tempfile.mktemp('','','') + os.rename(self.file_path, self.tmp_file_path) + + def __del__(self): + if os.path.isfile(self.tmp_file_path): + os.rename(self.tmp_file_path, self.file_path) + class IndexEntry(tuple): """ @@ -335,17 +352,13 @@ class Index(object): # move current index out of the way - otherwise the merge may fail # as it considers existing entries. moving it essentially clears the index. - # Unfortunately there is no 'soft' way to do it - cur_index = os.path.join(repo.path, 'index') - moved_index = os.path.join(repo.path, 'index_moved'+tempfile.mktemp('','','')) + # Unfortunately there is no 'soft' way to do it. + # The _TemporaryFileSwap assure the original file get put back + index_handler = _TemporaryFileSwap(os.path.join(repo.path, 'index')) try: - os.rename(cur_index, moved_index) repo.git.read_tree(*arg_list, **kwargs) index = cls.from_file(repo, tmp_index) finally: - # put back the original index first ! - if os.path.exists(moved_index): - os.rename(moved_index, cur_index) if os.path.exists(tmp_index): os.remove(tmp_index) # END index merge handling @@ -354,8 +367,10 @@ class Index(object): @classmethod def _index_mode_to_tree_index_mode(cls, index_mode): - """Cleanup a index_mode value. + """ + Cleanup a index_mode value. This will return a index_mode that can be stored in a tree object. + ``index_mode`` Index_mode to clean up. """ @@ -381,15 +396,29 @@ class Index(object): for entry in self.entries.itervalues(): mode = self._index_mode_to_tree_index_mode(entry.mode) blob = Blob(self.repo, entry.sha, mode, entry.path) + blob.size = entry.size output = (entry.stage, blob) if predicate(output): yield output # END for each entry - def write_tree(self, stream): + def write_tree(self): """ - Writes the + Writes the Index in self to a corresponding Tree file into the repository + object database and returns it as corresponding Tree object. + + Returns + Tree object representing this index """ - raise NotImplementedError("TODO") + index_path = os.path.join(self.repo.path, "index") + tmp_index_mover = _TemporaryFileSwap(index_path) + + self.to_file(self, index_path) + tree_sha = self.repo.git.write_tree() + + # remove our index file so that the original index can move back into place + # On linux it will silently overwrite, on windows it won't + os.remove(index_path) + return Tree(self.repo, tree_sha, 0, '') -- cgit v1.2.1 From aa921fee6014ef43bb2740240e9663e614e25662 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 00:32:16 +0200 Subject: Implemented merge/resolve handling , but realized that index writing is not yet working properly as it is sha1 checked as well. This explains what my 20 byte 'extension_data' actually is ;) --- lib/git/index.py | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 5bf654fe..d8ba6e18 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -126,6 +126,16 @@ class IndexEntry(tuple): @property def path(self): return self[10] + + + @classmethod + def from_blob(cls, blob): + """ + Returns + Minimal entry resembling the given blob objecft + """ + time = struct.pack(">LL", 0, 0) + return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) class Index(object): @@ -402,6 +412,61 @@ class Index(object): yield output # END for each entry + def unmerged_blobs(self): + """ + Returns + Iterator yielding dict(path : list( tuple( stage, Blob, ...))), being + a dictionary associating a path in the index with a list containing + stage/blob pairs + + Note: + Blobs that have been removed in one side simply do not exist in the + given stage. I.e. a file removed on the 'other' branch whose entries + are at stage 3 will not have a stage 3 entry. + """ + is_unmerged_blob = lambda t: t[0] != 0 + path_map = dict() + for stage, blob in self.iter_blobs(is_unmerged_blob): + path_map.setdefault(blob.path, list()).append((stage, blob)) + # END for each unmerged blob + + return path_map + + def resolve_blobs(self, iter_blobs): + """ + Resolve the blobs given in blob iterator. This will effectively remove the + index entries of the respective path at all non-null stages and add the given + blob as new stage null blob. + + For each path there may only be one blob, otherwise a ValueError will be raised + claiming the path is already at stage 0. + + Raise + ValueError if one of the blobs already existed at stage 0 + + Returns: + self + """ + for blob in iter_blobs: + stage_null_key = (blob.path, 0) + if stage_null_key in self.entries: + raise ValueError( "Blob %r already at stage 0" % blob ) + # END assert blob is not stage 0 already + + # delete all possible stages + for stage in (1, 2, 3): + try: + del( self.entries[(blob.path, stage)] ) + except KeyError: + pass + # END ignore key errors + # END for each possible stage + + self.entries[stage_null_key] = IndexEntry.from_blob(blob) + # END for each blob + + return self + def write_tree(self): """ Writes the Index in self to a corresponding Tree file into the repository @@ -414,11 +479,16 @@ class Index(object): tmp_index_mover = _TemporaryFileSwap(index_path) self.to_file(self, index_path) - tree_sha = self.repo.git.write_tree() - # remove our index file so that the original index can move back into place - # On linux it will silently overwrite, on windows it won't - os.remove(index_path) + try: + tree_sha = self.repo.git.write_tree() + finally: + # remove our index file so that the original index can move back into place + # On linux it will silently overwrite, on windows it won't + if os.path.isfile(index_path): + os.remove(index_path) + # END remove our own index file beforehand + # END write tree handling return Tree(self.repo, tree_sha, 0, '') -- cgit v1.2.1 From 30d822a468dc909aac5c83d078a59bfc85fc27aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 10:15:47 +0200 Subject: index writing now creates a sha on the content making it possible to write valid indices after manually removing or altering entriesgst --- lib/git/index.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index d8ba6e18..9a55da15 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -15,6 +15,7 @@ import tempfile import os import stat from git.objects import Blob, Tree +from git.utils import SHA1Writer class _TemporaryFileSwap(object): """ @@ -205,11 +206,20 @@ class Index(object): count = 0 while count < num_entries: entry = self._read_entry(stream) - self.entries[(entry.path,entry.stage)] = entry + self.entries[(entry.path, entry.stage)] = entry count += 1 # END for each entry - # this data chunk is the footer of the index, don't yet know what it is for + + # the footer contains extension data and a sha on the content so far + # Keep the extension footer,and verify we have a sha in the end self._extension_data = stream.read(~0) + assert len(self._extension_data) > 19, "Index Footer was not at least a sha on content as it was only %i bytes in size" % len(self._extension_data) + + content_sha = self._extension_data[-20:] + + # truncate the sha in the end as we will dynamically create it anyway + self._extension_data = self._extension_data[:-20] + @classmethod def from_file(cls, repo, file_path): @@ -296,6 +306,8 @@ class Index(object): Note Index writing based on the dulwich implementation """ + stream = SHA1Writer(stream) + # header stream.write("DIRC") stream.write(struct.pack(">LL", self.version, len(self.entries))) @@ -306,9 +318,13 @@ class Index(object): for entry in entries_sorted: self._write_cache_entry(stream, entry) # END for each entry - # write extension_data which we currently cannot interprete + + # write previously cached extensions data stream.write(self._extension_data) + # write the sha over the content + stream.write_sha() + @classmethod def from_tree(cls, repo, *treeish, **kwargs): -- cgit v1.2.1 From 3d9e7f1121d3bdbb08291c7164ad622350544a21 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 18:01:52 +0200 Subject: Index now behaves more like the default index if no explicit stream is given. It will lazily read its data on first access --- lib/git/index.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 9a55da15..fd5081ce 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -15,7 +15,8 @@ import tempfile import os import stat from git.objects import Blob, Tree -from git.utils import SHA1Writer +from git.utils import SHA1Writer, LazyMixin +from git.diff import Diffable class _TemporaryFileSwap(object): """ @@ -139,7 +140,7 @@ class IndexEntry(tuple): return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) -class Index(object): +class Index(LazyMixin): """ Implements an Index that can be manipulated using a native implementation in order to save git command function calls wherever possible. @@ -160,13 +161,28 @@ class Index(object): def __init__(self, repo, stream = None): """ Initialize this Index instance, optionally from the given ``stream`` + + If a stream is not given, the stream will be initialized from the current + repository's index on demand. """ self.repo = repo - self.entries = dict() self.version = self._VERSION self._extension_data = '' if stream is not None: self._read_from_stream(stream) + # END read from stream immediatly + + def _set_cache_(self, attr): + if attr == "entries": + # read the current index + fp = open(os.path.join(self.repo.path, "index"), "r") + try: + self._read_from_stream(fp) + finally: + fp.close() + # END read from default index on demand + else: + super(Index, self)._set_cache_(attr) @classmethod def _read_entry(cls, stream): @@ -197,13 +213,10 @@ class Index(object): def _read_from_stream(self, stream): """ Initialize this instance with index values read from the given stream - - Note - We explicitly do not clear the entries dict here to allow for reading - multiple chunks from multiple streams into the same Index instance """ self.version, num_entries = self._read_header(stream) count = 0 + self.entries = dict() while count < num_entries: entry = self._read_entry(stream) self.entries[(entry.path, entry.stage)] = entry @@ -298,7 +311,7 @@ class Index(object): Write the current state to the given stream ``stream`` - File-like object + File-like object. Returns self -- cgit v1.2.1 From 1496979cf7e9692ef869d2f99da6141756e08d25 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 19:43:01 +0200 Subject: default index writing now writes the index of the current repository in a fashion comparable to the native implementation --- lib/git/index.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index fd5081ce..4eabab15 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -15,7 +15,7 @@ import tempfile import os import stat from git.objects import Blob, Tree -from git.utils import SHA1Writer, LazyMixin +from git.utils import SHA1Writer, LazyMixin, ConcurrentWriteOperation from git.diff import Diffable class _TemporaryFileSwap(object): @@ -175,7 +175,7 @@ class Index(LazyMixin): def _set_cache_(self, attr): if attr == "entries": # read the current index - fp = open(os.path.join(self.repo.path, "index"), "r") + fp = open(self._index_path(), "r") try: self._read_from_stream(fp) finally: @@ -184,6 +184,9 @@ class Index(LazyMixin): else: super(Index, self)._set_cache_(attr) + def _index_path(self): + return os.path.join(self.repo.path, "index") + @classmethod def _read_entry(cls, stream): """Return: One entry of the given stream""" @@ -306,12 +309,14 @@ class Index(LazyMixin): real_size = ((stream.tell() - beginoffset + 8) & ~7) stream.write("\0" * ((beginoffset + real_size) - stream.tell())) - def write(self, stream): + def write(self, stream=None): """ - Write the current state to the given stream + Write the current state to the given stream or to the default repository + index. ``stream`` - File-like object. + File-like object or None. + If None, the default repository index will be overwritten. Returns self @@ -319,6 +324,13 @@ class Index(LazyMixin): Note Index writing based on the dulwich implementation """ + write_op = None + if stream is None: + write_op = ConcurrentWriteOperation(self._index_path()) + stream = write_op._begin_writing() + # stream = open(self._index_path() + # END stream handling + stream = SHA1Writer(stream) # header @@ -338,6 +350,9 @@ class Index(LazyMixin): # write the sha over the content stream.write_sha() + if write_op is not None: + write_op._end_writing() + @classmethod def from_tree(cls, repo, *treeish, **kwargs): @@ -504,7 +519,7 @@ class Index(LazyMixin): Returns Tree object representing this index """ - index_path = os.path.join(self.repo.path, "index") + index_path = self._index_path() tmp_index_mover = _TemporaryFileSwap(index_path) self.to_file(self, index_path) -- cgit v1.2.1 From ea33fe8b21d2b02f902b131aba0d14389f2f8715 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 22:14:02 +0200 Subject: Index: Is now diffable and appears to properly implement diffing against other items as well as the working tree Diff.Diffable: added callback allowing superclasses to preprocess diff arguments Diff.Diff: added eq, ne and hash methods, string methods would be nice --- lib/git/index.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 4eabab15..4217c9a2 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -14,9 +14,11 @@ import objects import tempfile import os import stat -from git.objects import Blob, Tree +import git.diff as diff + +from git.objects import Blob, Tree, Object from git.utils import SHA1Writer, LazyMixin, ConcurrentWriteOperation -from git.diff import Diffable + class _TemporaryFileSwap(object): """ @@ -140,7 +142,7 @@ class IndexEntry(tuple): return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) -class Index(LazyMixin): +class Index(LazyMixin, diff.Diffable): """ Implements an Index that can be manipulated using a native implementation in order to save git command function calls wherever possible. @@ -154,7 +156,7 @@ class Index(LazyMixin): The index contains an entries dict whose keys are tuples of type IndexEntry to facilitate access. """ - __slots__ = ( "repo", "version", "entries", "_extension_data" ) + __slots__ = ( "repo", "version", "entries", "_extension_data", "_is_default_index" ) _VERSION = 2 # latest version we support S_IFGITLINK = 0160000 @@ -168,10 +170,13 @@ class Index(LazyMixin): self.repo = repo self.version = self._VERSION self._extension_data = '' + self._is_default_index = True if stream is not None: + self._is_default_index = False self._read_from_stream(stream) # END read from stream immediatly + def _set_cache_(self, attr): if attr == "entries": # read the current index @@ -187,6 +192,18 @@ class Index(LazyMixin): def _index_path(self): return os.path.join(self.repo.path, "index") + + @property + def path(self): + """ + Returns + Path to the index file we are representing or None if we are + a loose index that was read from a stream. + """ + if self._is_default_index: + return self._index_path() + return None + @classmethod def _read_entry(cls, stream): """Return: One entry of the given stream""" @@ -535,4 +552,51 @@ class Index(LazyMixin): # END write tree handling return Tree(self.repo, tree_sha, 0, '') + + def _process_diff_args(self, args): + try: + args.pop(args.index(self)) + except IndexError: + pass + # END remove self + return args + + def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): + """ + Diff this index against the working copy or a Tree or Commit object + + For a documentation of the parameters and return values, see + Diffable.diff + + Note + Will only work with indices that represent the default git index as + they have not been initialized with a stream. + """ + if not self._is_default_index: + raise AssertionError( "Cannot diff custom indices as they do not represent the default git index" ) + + # index against index is always empty + if other is self.Index: + return diff.DiffIndex() + + # index against anything but None is a reverse diff with the respective + # item. Handle existing -R flags properly. Transform strings to the object + # so that we can call diff on it + if isinstance(other, basestring): + other = Object.new(self.repo, other) + # END object conversion + + if isinstance(other, Object): + # invert the existing R flag + cur_val = kwargs.get('R', False) + kwargs['R'] = not cur_val + return other.diff(self.Index, paths, create_patch, **kwargs) + # END diff against other item handlin + + # if other is not None here, something is wrong + if other is not None: + raise ValueError( "other must be None, Diffable.Index, a Tree or Commit, was %r" % other ) + + # diff against working copy - can be handled by superclass natively + return super(Index, self).diff(other, paths, create_patch, **kwargs) -- cgit v1.2.1 From a10aa36bc6a82bd50df6f3df7d6b7ce04a7070f1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 10:42:26 +0200 Subject: Renamed Index to IndexFile, adjusted tests, it will only operate on physical files, not on streams, as Indices are not streamed by any git command ( at least not in raw format ) --- lib/git/index.py | 150 ++++++++++++++++++------------------------------------- 1 file changed, 49 insertions(+), 101 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 4217c9a2..5f13790c 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -13,6 +13,7 @@ import mmap import objects import tempfile import os +import sys import stat import git.diff as diff @@ -34,7 +35,10 @@ class _TemporaryFileSwap(object): def __del__(self): if os.path.isfile(self.tmp_file_path): + if sys.platform == "win32" and os.path.exists(self.file_path): + os.remove(self.file_path) os.rename(self.tmp_file_path, self.file_path) + # END temp file exists class IndexEntry(tuple): @@ -142,7 +146,7 @@ class IndexEntry(tuple): return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) -class Index(LazyMixin, diff.Diffable): +class IndexFile(LazyMixin, diff.Diffable): """ Implements an Index that can be manipulated using a native implementation in order to save git command function calls wherever possible. @@ -155,14 +159,20 @@ class Index(LazyMixin, diff.Diffable): ``Entries`` The index contains an entries dict whose keys are tuples of type IndexEntry to facilitate access. + + As opposed to the Index type, the IndexFile represents the index on file level. + This can be considered an alternate, file-based implementation of the Index class + with less support for common functions. Use it for very special and custom index + handling. """ - __slots__ = ( "repo", "version", "entries", "_extension_data", "_is_default_index" ) + __slots__ = ( "repo", "version", "entries", "_extension_data", "_file_path" ) _VERSION = 2 # latest version we support S_IFGITLINK = 0160000 - def __init__(self, repo, stream = None): + def __init__(self, repo, file_path=None): """ - Initialize this Index instance, optionally from the given ``stream`` + Initialize this Index instance, optionally from the given ``file_path``. + If no file_path is given, we will be created from the current index file. If a stream is not given, the stream will be initialized from the current repository's index on demand. @@ -170,24 +180,27 @@ class Index(LazyMixin, diff.Diffable): self.repo = repo self.version = self._VERSION self._extension_data = '' - self._is_default_index = True - if stream is not None: - self._is_default_index = False - self._read_from_stream(stream) - # END read from stream immediatly - + self._file_path = file_path or self._index_path() def _set_cache_(self, attr): if attr == "entries": # read the current index - fp = open(self._index_path(), "r") + # try memory map for speed + fp = open(self._file_path, "r") + stream = fp try: - self._read_from_stream(fp) + stream = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) + except Exception: + pass + # END memory mapping + + try: + self._read_from_stream(stream) finally: fp.close() # END read from default index on demand else: - super(Index, self)._set_cache_(attr) + super(IndexFile, self)._set_cache_(attr) def _index_path(self): return os.path.join(self.repo.path, "index") @@ -197,12 +210,9 @@ class Index(LazyMixin, diff.Diffable): def path(self): """ Returns - Path to the index file we are representing or None if we are - a loose index that was read from a stream. + Path to the index file we are representing """ - if self._is_default_index: - return self._index_path() - return None + return self._file_path @classmethod def _read_entry(cls, stream): @@ -252,62 +262,8 @@ class Index(LazyMixin, diff.Diffable): # truncate the sha in the end as we will dynamically create it anyway self._extension_data = self._extension_data[:-20] - - @classmethod - def from_file(cls, repo, file_path): - """ - Returns - Index instance as recreated from the given stream. - - ``repo`` - Repository the index is related to - - ``file_pa `` - File path pointing to git index file - - Note - Reading is based on the dulwich project. - """ - fp = open(file_path, "r") - - # try memory map for speed - stream = fp - try: - stream = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) - except Exception: - pass - # END memory mapping - - try: - return cls(repo, stream) - finally: - fp.close() - - @classmethod - def to_file(cls, index, file_path): - """ - Write the index data to the given file path. - - ``index`` - Index you wish to write. - - ``file_path`` - Path at which to write the index data. Please note that missing directories - will lead to an exception to be thrown. - - Raise - IOError if the file could not be written - """ - fp = open(file_path, "w") - try: - return index.write(fp) - finally: - fp.close() - # END exception handling - - @classmethod def _write_cache_entry(cls, stream, entry): """ @@ -326,14 +282,14 @@ class Index(LazyMixin, diff.Diffable): real_size = ((stream.tell() - beginoffset + 8) & ~7) stream.write("\0" * ((beginoffset + real_size) - stream.tell())) - def write(self, stream=None): + def write(self, file_path = None): """ - Write the current state to the given stream or to the default repository - index. + Write the current state to our file path or to the given one - ``stream`` - File-like object or None. - If None, the default repository index will be overwritten. + ``file_path`` + If None, we will write to our stored file path from which we have + been initialized. Otherwise we write to the given file path. + Please note that this will not change the file_path of this index. Returns self @@ -341,12 +297,8 @@ class Index(LazyMixin, diff.Diffable): Note Index writing based on the dulwich implementation """ - write_op = None - if stream is None: - write_op = ConcurrentWriteOperation(self._index_path()) - stream = write_op._begin_writing() - # stream = open(self._index_path() - # END stream handling + write_op = ConcurrentWriteOperation(file_path or self._file_path) + stream = write_op._begin_writing() stream = SHA1Writer(stream) @@ -366,9 +318,7 @@ class Index(LazyMixin, diff.Diffable): # write the sha over the content stream.write_sha() - - if write_op is not None: - write_op._end_writing() + write_op._end_writing() @classmethod @@ -394,6 +344,11 @@ class Index(LazyMixin, diff.Diffable): ``**kwargs`` Additional arguments passed to git-read-tree + Returns + New IndexFile instance. It will point to a temporary index location which + does not exist anymore. If you intend to write such a merged Index, supply + an alternate file_path to its 'write' method. + Note: In the three-way merge case, --aggressive will be specified to automatically resolve more cases in a commonly correct manner. Specify trivial=True as kwarg @@ -428,7 +383,8 @@ class Index(LazyMixin, diff.Diffable): index_handler = _TemporaryFileSwap(os.path.join(repo.path, 'index')) try: repo.git.read_tree(*arg_list, **kwargs) - index = cls.from_file(repo, tmp_index) + index = cls(repo, tmp_index) + index.entries # force it to read the file finally: if os.path.exists(tmp_index): os.remove(tmp_index) @@ -539,20 +495,11 @@ class Index(LazyMixin, diff.Diffable): index_path = self._index_path() tmp_index_mover = _TemporaryFileSwap(index_path) - self.to_file(self, index_path) + self.write(index_path) + tree_sha = self.repo.git.write_tree() - try: - tree_sha = self.repo.git.write_tree() - finally: - # remove our index file so that the original index can move back into place - # On linux it will silently overwrite, on windows it won't - if os.path.isfile(index_path): - os.remove(index_path) - # END remove our own index file beforehand - # END write tree handling return Tree(self.repo, tree_sha, 0, '') - def _process_diff_args(self, args): try: args.pop(args.index(self)) @@ -572,7 +519,8 @@ class Index(LazyMixin, diff.Diffable): Will only work with indices that represent the default git index as they have not been initialized with a stream. """ - if not self._is_default_index: + # perhaps we shouldn't diff these at all, or we swap them in place first + if self._file_path != self._index_path(): raise AssertionError( "Cannot diff custom indices as they do not represent the default git index" ) # index against index is always empty @@ -598,5 +546,5 @@ class Index(LazyMixin, diff.Diffable): raise ValueError( "other must be None, Diffable.Index, a Tree or Commit, was %r" % other ) # diff against working copy - can be handled by superclass natively - return super(Index, self).diff(other, paths, create_patch, **kwargs) + return super(IndexFile, self).diff(other, paths, create_patch, **kwargs) -- cgit v1.2.1 From 3d3a24b22d340c62fafc0e75a349c0ffe34d99d7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 11:07:04 +0200 Subject: Added repo.index property including simple test, and additional ideas in the TODO list --- lib/git/index.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 5f13790c..5452708d 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -289,7 +289,8 @@ class IndexFile(LazyMixin, diff.Diffable): ``file_path`` If None, we will write to our stored file path from which we have been initialized. Otherwise we write to the given file path. - Please note that this will not change the file_path of this index. + Please note that this will change the file_path of this index to + the one you gave. Returns self @@ -319,7 +320,10 @@ class IndexFile(LazyMixin, diff.Diffable): # write the sha over the content stream.write_sha() write_op._end_writing() - + + # make sure we represent what we have written + if file_path is not None: + self._file_path = file_path @classmethod def from_tree(cls, repo, *treeish, **kwargs): -- cgit v1.2.1 From a7a4388eeaa4b6b94192dce67257a34c4a6cbd26 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 12:14:22 +0200 Subject: Added frame for IndexFile add/remove/commit methods and respective test markers --- lib/git/index.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 5452708d..9fd1fad2 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -146,6 +146,22 @@ class IndexEntry(tuple): return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) +def default_index(func): + """ + Decorator assuring the wrapped method may only run if we are the default + repository index. This is as we rely on git commands that operate + on that index only. + """ + def check_default_index(self, *args, **kwargs): + if self._file_path != self._index_path(): + raise AssertionError( "Cannot call %r on indices that do not represent the default git index" % func.__name__ ) + return func(self, *args, **kwargs) + # END wrpaper method + + check_default_index.__name__ = func.__name__ + return check_default_index + + class IndexFile(LazyMixin, diff.Diffable): """ Implements an Index that can be manipulated using a native implementation in @@ -487,6 +503,22 @@ class IndexFile(LazyMixin, diff.Diffable): # END for each blob return self + + def update(self): + """ + Reread the contents of our index file, discarding all cached information + we might have. + + Note: + This is a possibly dangerious operations as it will discard your changes + to index.endtries + + Returns + self + """ + del(self.entries) + self.entries + return self def write_tree(self): """ @@ -512,6 +544,88 @@ class IndexFile(LazyMixin, diff.Diffable): # END remove self return args + @default_index + def add(self, items, **kwargs): + """ + Add files from the working copy, specific blobs or IndexEntries + to the index. + + TODO: Its important to specify a way to add symlinks directly, even + on systems that do not support it, like ... erm ... windows. + + ``**kwargs`` + Additional keyword arguments to be passed to git-update-index + + Returns + List(IndexEntries) representing the entries just added + """ + raise NotImplementedError("todo") + + @default_index + def remove(self, items, affect_working_tree=False, **kwargs): + """ + Remove the given file_paths or blobs from the index and optionally from + the working tree as well. + + ``items`` + TODO + + ``affect_working_tree`` + If True, the entry will also be removed from the working tree, physically + removing the respective file. This may fail if there are uncommited changes + in it. + + ``**kwargs`` + Additional keyword arguments to be passed to git-update-index + + Returns + self + """ + raise NotImplementedError("todo") + return self + + @default_index + def commit(self, message=None, parent_commits=None, **kwargs): + """ + Commit the current index, creating a commit object. + + ``message`` + Commit message + + ``parent_commits`` + Optional Commit objects to use as parents for the new commit. + If None or empty, the current head commit will be the parent of the + new commit object + + ``**kwargs`` + Additional keyword arguments passed to git-commit + + Returns + Commit object representing the new commit + """ + raise NotImplementedError("todo") + + @default_index + def reset(self, commit='HEAD', working_tree=False, **kwargs): + """ + Reset the index to reflect the tree at the given commit. This will not + adjust our HEAD reference as opposed to HEAD.reset. + + ``commit`` + Revision, Reference or Commit specifying the commit we should represent. + If you want to specify a tree only, use IndexFile.from_tree and overwrite + the default index. + + ``working_tree`` + If True, the files in the working tree will reflect the changed index. + If False, the working tree will not be touched + + ``**kwargs`` + Additional keyword arguments passed to git-reset + """ + raise NotImplementedError("todo") + + @default_index def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): """ Diff this index against the working copy or a Tree or Commit object @@ -523,10 +637,6 @@ class IndexFile(LazyMixin, diff.Diffable): Will only work with indices that represent the default git index as they have not been initialized with a stream. """ - # perhaps we shouldn't diff these at all, or we swap them in place first - if self._file_path != self._index_path(): - raise AssertionError( "Cannot diff custom indices as they do not represent the default git index" ) - # index against index is always empty if other is self.Index: return diff.DiffIndex() -- cgit v1.2.1 From 81d8788d40867f4e5cf3016ef219473951a7f6ed Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 17:19:36 +0200 Subject: IndexEntry is now based on a 'minimal' version that is suitable to be fed into UpdateIndex. The Inode and device information is only needed to quickly compare the index against the working tree for changes, hence it should not be that dominant in the API either. More changes to come --- lib/git/index.py | 121 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 52 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 9fd1fad2..754c9f15 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -41,7 +41,57 @@ class _TemporaryFileSwap(object): # END temp file exists -class IndexEntry(tuple): +class BaseIndexEntry(tuple): + """ + Small Brother of an index entry which can be created to describe changes + done to the index in which case plenty of additional information is not requried. + + As the first 4 data members match exactly to the IndexEntry type, methods + expecting a BaseIndexEntry can also handle full IndexEntries even if they + use numeric indices for performance reasons. + """ + + @property + def mode(self): + """ + File Mode, compatible to stat module constants + """ + return self[0] + + @property + def sha(self): + """ + hex sha of the blob + """ + return self[1] + + @property + def stage(self): + """ + Stage of the entry, either: + 0 = default stage + 1 = stage before a merge or common ancestor entry in case of a 3 way merge + 2 = stage of entries from the 'left' side of the merge + 3 = stage of entries from the right side of the merge + Note: + For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html + """ + return self[2] + + @property + def path(self): + return self[3] + + @classmethod + def from_blob(cls, blob, stage = 0): + """ + Returns + Fully equipped BaseIndexEntry at the given stage + """ + return cls((blob.mode, blob.id, stage, blob.path)) + + +class IndexEntry(BaseIndexEntry): """ Allows convenient access to IndexEntry data without completely unpacking it. @@ -57,49 +107,42 @@ class IndexEntry(tuple): Tuple(int_time_seconds_since_epoch, int_nano_seconds) of the file's creation time """ - return struct.unpack(">LL", self[0]) + return struct.unpack(">LL", self[4]) @property def mtime(self): """ See ctime property, but returns modification time """ - return struct.unpack(">LL", self[1]) + return struct.unpack(">LL", self[5]) @property def dev(self): """ Device ID """ - return self[2] + return self[6] @property def inode(self): """ Inode ID """ - return self[3] - - @property - def mode(self): - """ - File Mode, compatible to stat module constants - """ - return self[4] - + return self[7] + @property def uid(self): """ User ID """ - return self[5] + return self[8] @property def gid(self): """ Group ID """ - return self[6] + return self[9] @property def size(self): @@ -109,33 +152,8 @@ class IndexEntry(tuple): Note Will be 0 if the stage is not 0 ( hence it is an unmerged entry ) """ - return self[7] - - @property - def sha(self): - """ - hex sha of the blob - """ - return self[8] - - @property - def stage(self): - """ - Stage of the entry, either: - 0 = default stage - 1 = stage before a merge or common ancestor entry in case of a 3 way merge - 2 = stage of entries from the 'left' side of the merge - 3 = stage of entries from the right side of the merge - Note: - For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html - """ - return self[9] - - @property - def path(self): return self[10] - @classmethod def from_blob(cls, blob): """ @@ -143,7 +161,7 @@ class IndexEntry(tuple): Minimal entry resembling the given blob objecft """ time = struct.pack(">LL", 0, 0) - return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) + return IndexEntry((blob.mode, blob.id, 0, blob.path, time, time, 0, 0, 0, 0, blob.size)) def default_index(func): @@ -243,8 +261,7 @@ class IndexFile(LazyMixin, diff.Diffable): real_size = ((stream.tell() - beginoffset + 8) & ~7) data = stream.read((beginoffset + real_size) - stream.tell()) - return IndexEntry((ctime, mtime, dev, ino, mode, uid, gid, size, - binascii.hexlify(sha), flags >> 12, path)) + return IndexEntry((mode, binascii.hexlify(sha), flags >> 12, path, ctime, mtime, dev, ino, uid, gid, size)) @classmethod def _read_header(cls, stream): @@ -286,14 +303,14 @@ class IndexFile(LazyMixin, diff.Diffable): Write an IndexEntry to a stream """ beginoffset = stream.tell() - stream.write(entry[0]) # ctime - stream.write(entry[1]) # mtime - path = entry[10] + stream.write(entry[4]) # ctime + stream.write(entry[5]) # mtime + path = entry[3] plen = len(path) & 0x0fff # path length - assert plen == len(path), "Path %s too long to fit into index" % entry[10] - flags = plen | (entry[9] << 12)# stage and path length are 2 byte flags - stream.write(struct.pack(">LLLLLL20sH", entry[2], entry[3], entry[4], - entry[5], entry[6], entry[7], binascii.unhexlify(entry[8]), flags)) + assert plen == len(path), "Path %s too long to fit into index" % entry[3] + flags = plen | (entry[2] << 12)# stage and path length are 2 byte flags + stream.write(struct.pack(">LLLLLL20sH", entry[6], entry[7], entry[0], + entry[8], entry[9], entry[10], binascii.unhexlify(entry[1]), flags)) stream.write(path) real_size = ((stream.tell() - beginoffset + 8) & ~7) stream.write("\0" * ((beginoffset + real_size) - stream.tell())) @@ -325,7 +342,7 @@ class IndexFile(LazyMixin, diff.Diffable): # body entries_sorted = self.entries.values() - entries_sorted.sort(key=lambda e: (e[10], e[9])) # use path/stage as sort key + entries_sorted.sort(key=lambda e: (e[3], e[2])) # use path/stage as sort key for entry in entries_sorted: self._write_cache_entry(stream, entry) # END for each entry @@ -623,7 +640,7 @@ class IndexFile(LazyMixin, diff.Diffable): ``**kwargs`` Additional keyword arguments passed to git-reset """ - raise NotImplementedError("todo") + raise NotImplementedError("todo: use git-read-tree if there is no working tree to update") @default_index def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): -- cgit v1.2.1 From 13a26d4f9c22695033040dfcd8c76fd94187035b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 17:55:33 +0200 Subject: Implemented index.reset method including test --- lib/git/index.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 754c9f15..111dc8d2 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -528,13 +528,13 @@ class IndexFile(LazyMixin, diff.Diffable): Note: This is a possibly dangerious operations as it will discard your changes - to index.endtries + to index.entries Returns self """ del(self.entries) - self.entries + # allows to lazily reread on demand return self def write_tree(self): @@ -623,7 +623,7 @@ class IndexFile(LazyMixin, diff.Diffable): raise NotImplementedError("todo") @default_index - def reset(self, commit='HEAD', working_tree=False, **kwargs): + def reset(self, commit='HEAD', working_tree=False, paths=None, **kwargs): """ Reset the index to reflect the tree at the given commit. This will not adjust our HEAD reference as opposed to HEAD.reset. @@ -636,11 +636,23 @@ class IndexFile(LazyMixin, diff.Diffable): ``working_tree`` If True, the files in the working tree will reflect the changed index. If False, the working tree will not be touched + Please note that changes to the working copy will be discarded without + warning ! ``**kwargs`` Additional keyword arguments passed to git-reset + + Returns + self """ - raise NotImplementedError("todo: use git-read-tree if there is no working tree to update") + head = self.repo.head + prev_commit = head.commit + + # reset to get the tree/working copy + head.reset(commit, index=True, working_tree=working_tree, paths=paths, **kwargs) + # put the head back + head.reset(prev_commit, index=False, working_tree=False) + return self @default_index def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): -- cgit v1.2.1 From 0cd09bd306486028f5442c56ef2e947355a06282 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 21:49:13 +0200 Subject: index.remove implemented including throrough test --- lib/git/index.py | 152 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 131 insertions(+), 21 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 111dc8d2..65e658b0 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -164,6 +164,26 @@ class IndexEntry(BaseIndexEntry): return IndexEntry((blob.mode, blob.id, 0, blob.path, time, time, 0, 0, 0, 0, blob.size)) +def clear_cache(func): + """ + Decorator for functions that alter the index using the git command. This would + invalidate our possibly existing entries dictionary which is why it must be + deleted to allow it to be lazily reread later. + + Note + This decorator will not be required once all functions are implemented + natively which in fact is possible, but probably not feasible performance wise. + """ + def clear_cache_if_not_raised(self, *args, **kwargs): + rval = func(self, *args, **kwargs) + del(self.entries) + return rval + + # END wrapper method + clear_cache_if_not_raised.__name__ = func.__name__ + return clear_cache_if_not_raised + + def default_index(func): """ Decorator assuring the wrapped method may only run if we are the default @@ -194,10 +214,8 @@ class IndexFile(LazyMixin, diff.Diffable): The index contains an entries dict whose keys are tuples of type IndexEntry to facilitate access. - As opposed to the Index type, the IndexFile represents the index on file level. - This can be considered an alternate, file-based implementation of the Index class - with less support for common functions. Use it for very special and custom index - handling. + You may only read the entries dict or manipulate it through designated methods. + Otherwise changes to it will be lost when changing the index using its methods. """ __slots__ = ( "repo", "version", "entries", "_extension_data", "_file_path" ) _VERSION = 2 # latest version we support @@ -500,6 +518,10 @@ class IndexFile(LazyMixin, diff.Diffable): Returns: self + + Note + You will have to write the index manually once you are done, i.e. + index.resolve_blobs(blobs).write() """ for blob in iter_blobs: stage_null_key = (blob.path, 0) @@ -561,45 +583,132 @@ class IndexFile(LazyMixin, diff.Diffable): # END remove self return args + + def _to_relative_path(self, path): + """ + Return + Version of path relative to our git directory or raise ValueError + if it is not within our git direcotory + """ + if not os.path.isabs(path): + return path + relative_path = path.replace(self.repo.git.git_dir+"/", "") + if relative_path == path: + raise ValueError("Absolute path %r is not in git repository at %r" % (path,self.repo.git.git_dir)) + return relative_path + + @clear_cache @default_index def add(self, items, **kwargs): """ - Add files from the working copy, specific blobs or IndexEntries - to the index. - - TODO: Its important to specify a way to add symlinks directly, even - on systems that do not support it, like ... erm ... windows. + Add files from the working tree, specific blobs or BaseIndexEntries + to the index. The underlying index file will be written immediately, hence + you should provide as many items as possible to minimize the amounts of writes + ``items`` + Multiple types of items are supported, types can be mixed within one call. + Different types imply a different handling. File paths may generally be + relative or absolute. + + - path string + strings denote a relative or absolute path into the repository pointing to + an existing file, i.e. CHANGES, lib/myfile.ext, /home/gitrepo/lib/myfile.ext. + + Paths provided like this must exist. When added, they will be written + into the object database. + + This equals a straight git-add. + + They are added at stage 0 + + - Blob object + Blobs are added as they are assuming a valid mode is set. + The file they refer to may or may not exist in the file system + + If their sha is null ( 40*0 ), their path must exist in the file system + as an object will be created from the data at the path.The handling + now very much equals the way string paths are processed, except that + the mode you have set will be kept. This allows you to create symlinks + by settings the mode respectively and writing the target of the symlink + directly into the file. This equals a default Linux-Symlink which + is not dereferenced automatically, except that it can be created on + filesystems not supporting it as well. + + They are added at stage 0 + + - BaseIndexEntry or type + Handling equals the one of Blob objects, but the stage may be + explicitly set. + ``**kwargs`` - Additional keyword arguments to be passed to git-update-index + Additional keyword arguments to be passed to git-update-index, such + as index_only. Returns - List(IndexEntries) representing the entries just added + List(BaseIndexEntries) representing the entries just actually added. """ raise NotImplementedError("todo") + @clear_cache @default_index - def remove(self, items, affect_working_tree=False, **kwargs): + def remove(self, items, working_tree=False, **kwargs): """ - Remove the given file_paths or blobs from the index and optionally from + Remove the given items from the index and optionally from the working tree as well. ``items`` - TODO + Multiple types of items are supported which may be be freely mixed. + + - path string + Remove the given path at all stages. If it is a directory, you must + specify the r=True keyword argument to remove all file entries + below it. If absolute paths are given, they will be converted + to a path relative to the git repository directory containing + the working tree + + The path string may include globs, such as *.c. + + - Blob object + Only the path portion is used in this case. + + - BaseIndexEntry or compatible type + The only relevant information here Yis the path. The stage is ignored. - ``affect_working_tree`` + ``working_tree`` If True, the entry will also be removed from the working tree, physically removing the respective file. This may fail if there are uncommited changes in it. ``**kwargs`` - Additional keyword arguments to be passed to git-update-index + Additional keyword arguments to be passed to git-rm, such + as 'r' to allow recurive removal of Returns - self - """ - raise NotImplementedError("todo") - return self + List(path_string, ...) list of paths that have been removed effectively. + This is interesting to know in case you have provided a directory or + globs. Paths are relative to the + """ + args = list() + if not working_tree: + args.append("--cached") + args.append("--") + + # preprocess paths + paths = list() + for item in items: + if isinstance(item, (BaseIndexEntry,Blob)): + paths.append(self._to_relative_path(item.path)) + elif isinstance(item, basestring): + paths.append(self._to_relative_path(item)) + else: + raise TypeError("Invalid item type: %r" % item) + # END for each item + + removed_paths = self.repo.git.rm(args, paths, **kwargs).splitlines() + + # process output to gain proper paths + # rm 'path' + return [ p[4:-1] for p in removed_paths ] @default_index def commit(self, message=None, parent_commits=None, **kwargs): @@ -621,7 +730,8 @@ class IndexFile(LazyMixin, diff.Diffable): Commit object representing the new commit """ raise NotImplementedError("todo") - + + @clear_cache @default_index def reset(self, commit='HEAD', working_tree=False, paths=None, **kwargs): """ -- cgit v1.2.1 From f9cec00938d9059882bb8eabdaf2f775943e00e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Oct 2009 00:08:33 +0200 Subject: index.commit: implemented initial version, but in fact some more changes are required to have a nice API. Tests are not yet fully done either --- lib/git/index.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 65e658b0..86bdfd39 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -17,7 +17,7 @@ import sys import stat import git.diff as diff -from git.objects import Blob, Tree, Object +from git.objects import Blob, Tree, Object, Commit from git.utils import SHA1Writer, LazyMixin, ConcurrentWriteOperation @@ -711,25 +711,51 @@ class IndexFile(LazyMixin, diff.Diffable): return [ p[4:-1] for p in removed_paths ] @default_index - def commit(self, message=None, parent_commits=None, **kwargs): + def commit(self, message, parent_commits=None): """ Commit the current index, creating a commit object. ``message`` - Commit message + Commit message. It may be an empty string if no message is provided. + It will be converted to a string in any case. ``parent_commits`` - Optional Commit objects to use as parents for the new commit. - If None or empty, the current head commit will be the parent of the + Optional Commit objects to use as parents for the new commit. + If empty list, the commit will have no parents at all and become + a root commit. + If None , the current head commit will be the parent of the new commit object - ``**kwargs`` - Additional keyword arguments passed to git-commit - Returns Commit object representing the new commit + + Note: + Additional information about hte committer and Author are taken from the + environment or from the git configuration, see git-commit-tree for + more information """ - raise NotImplementedError("todo") + parents = parent_commits + if parent_commits is None: + parent_commits = [ self.repo.head.commit ] + + parent_args = [ ("-p", str(commit)) for commit in parent_commits ] + + # create message stream + tmp_file_path = tempfile.mktemp() + fp = open(tmp_file_path,"w") + fp.write(str(message)) + fp.close() + fp = open(tmp_file_path,"r") + fp.seek(0) + + try: + # write the current index as tree + tree_sha = self.repo.git.write_tree() + commit_sha = self.repo.git.commit_tree(tree_sha, parent_args, istream=fp) + return Commit(self.repo, commit_sha) + finally: + fp.close() + os.remove(tmp_file_path) @clear_cache @default_index -- cgit v1.2.1 From 5ba2aef8f59009756567a53daaf918afa851c304 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Oct 2009 11:25:45 +0100 Subject: added head kwarg to reset and commit method, allowing to automatically change the head to the given commit, which makes the methods more versatile --- lib/git/index.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 86bdfd39..89e716d4 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -711,7 +711,7 @@ class IndexFile(LazyMixin, diff.Diffable): return [ p[4:-1] for p in removed_paths ] @default_index - def commit(self, message, parent_commits=None): + def commit(self, message, parent_commits=None, head=True): """ Commit the current index, creating a commit object. @@ -726,6 +726,11 @@ class IndexFile(LazyMixin, diff.Diffable): If None , the current head commit will be the parent of the new commit object + ``head`` + If True, the HEAD will be advanced to the new commit automatically. + Else the HEAD will remain pointing on the previous commit. This could + lead to undesired results when diffing files. + Returns Commit object representing the new commit @@ -752,17 +757,23 @@ class IndexFile(LazyMixin, diff.Diffable): # write the current index as tree tree_sha = self.repo.git.write_tree() commit_sha = self.repo.git.commit_tree(tree_sha, parent_args, istream=fp) - return Commit(self.repo, commit_sha) + new_commit = Commit(self.repo, commit_sha) + + if head: + self.repo.head.commit = new_commit + # END advance head handling + + return new_commit finally: fp.close() os.remove(tmp_file_path) - + @clear_cache @default_index - def reset(self, commit='HEAD', working_tree=False, paths=None, **kwargs): + def reset(self, commit='HEAD', working_tree=False, paths=None, head=False, **kwargs): """ Reset the index to reflect the tree at the given commit. This will not - adjust our HEAD reference as opposed to HEAD.reset. + adjust our HEAD reference as opposed to HEAD.reset by default. ``commit`` Revision, Reference or Commit specifying the commit we should represent. @@ -775,19 +786,27 @@ class IndexFile(LazyMixin, diff.Diffable): Please note that changes to the working copy will be discarded without warning ! + ``head`` + If True, the head will be set to the given commit. This is False by default, + but if True, this method behaves like HEAD.reset. + ``**kwargs`` Additional keyword arguments passed to git-reset Returns self """ - head = self.repo.head - prev_commit = head.commit + cur_head = self.repo.head + prev_commit = cur_head.commit # reset to get the tree/working copy - head.reset(commit, index=True, working_tree=working_tree, paths=paths, **kwargs) - # put the head back - head.reset(prev_commit, index=False, working_tree=False) + cur_head.reset(commit, index=True, working_tree=working_tree, paths=paths, **kwargs) + + # put the head back, possibly + if not head: + cur_head.reset(prev_commit, index=False, working_tree=False) + # END reset head + return self @default_index -- cgit v1.2.1 From 0ef1f89abe5b2334705ee8f1a6da231b0b6c9a50 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Oct 2009 22:37:48 +0100 Subject: index.add: Finished implemenation including through tests index.checkout: added simple method allowing to checkout files from the index, including simple test --- lib/git/index.py | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 142 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 89e716d4..cc3f3a4e 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -15,6 +15,7 @@ import tempfile import os import sys import stat +import subprocess import git.diff as diff from git.objects import Blob, Tree, Object, Commit @@ -51,6 +52,9 @@ class BaseIndexEntry(tuple): use numeric indices for performance reasons. """ + def __str__(self): + return "%o %s %i\t%s\n" % (self.mode, self.sha, self.stage, self.path) + @property def mode(self): """ @@ -466,6 +470,12 @@ class IndexFile(LazyMixin, diff.Diffable): ret |= (index_mode & 0111) return ret + @classmethod + def _tree_mode_to_index_mode(cls, tree_mode): + """ + Convert a tree mode to index mode as good as possible + """ + def iter_blobs(self, predicate = lambda t: True): """ Returns @@ -597,9 +607,29 @@ class IndexFile(LazyMixin, diff.Diffable): raise ValueError("Absolute path %r is not in git repository at %r" % (path,self.repo.git.git_dir)) return relative_path + def _preprocess_add_items(self, items): + """ + Split the items into two lists of path strings and BaseEntries. + """ + paths = list() + entries = list() + + for item in items: + if isinstance(item, basestring): + paths.append(self._to_relative_path(item)) + elif isinstance(item, Blob): + entries.append(BaseIndexEntry.from_blob(item)) + elif isinstance(item, BaseIndexEntry): + entries.append(item) + else: + raise TypeError("Invalid Type: %r" % item) + # END for each item + return (paths, entries) + + @clear_cache @default_index - def add(self, items, **kwargs): + def add(self, items, force=True, **kwargs): """ Add files from the working tree, specific blobs or BaseIndexEntries to the index. The underlying index file will be written immediately, hence @@ -612,18 +642,23 @@ class IndexFile(LazyMixin, diff.Diffable): - path string strings denote a relative or absolute path into the repository pointing to - an existing file, i.e. CHANGES, lib/myfile.ext, /home/gitrepo/lib/myfile.ext. + an existing file, i.e. CHANGES, lib/myfile.ext, '/home/gitrepo/lib/myfile.ext'. Paths provided like this must exist. When added, they will be written into the object database. + PathStrings may contain globs, such as 'lib/__init__*' or can be directories + like 'lib', the latter ones will add all the files within the dirctory and + subdirectories. + This equals a straight git-add. They are added at stage 0 - Blob object Blobs are added as they are assuming a valid mode is set. - The file they refer to may or may not exist in the file system + The file they refer to may or may not exist in the file system, but + must be a path relative to our repository. If their sha is null ( 40*0 ), their path must exist in the file system as an object will be created from the data at the path.The handling @@ -634,12 +669,21 @@ class IndexFile(LazyMixin, diff.Diffable): is not dereferenced automatically, except that it can be created on filesystems not supporting it as well. + Please note that globs or directories are not allowed in Blob objects. + They are added at stage 0 - BaseIndexEntry or type Handling equals the one of Blob objects, but the stage may be explicitly set. + ``force`` + If True, otherwise ignored or excluded files will be + added anyway. + As opposed to the git-add command, we enable this flag by default + as the API user usually wants the item to be added even though + they might be excluded. + ``**kwargs`` Additional keyword arguments to be passed to git-update-index, such as index_only. @@ -647,7 +691,54 @@ class IndexFile(LazyMixin, diff.Diffable): Returns List(BaseIndexEntries) representing the entries just actually added. """ - raise NotImplementedError("todo") + # sort the entries into strings and Entries, Blobs are converted to entries + # automatically + # paths can be git-added, for everything else we use git-update-index + entries_added = list() + paths, entries = self._preprocess_add_items(items) + + if paths: + git_add_output = self.repo.git.add(paths, v=True) + # force rereading our entries + del(self.entries) + for line in git_add_output.splitlines(): + # line contains: + # add '' + added_file = line[5:-1] + entries_added.append(self.entries[(added_file,0)]) + # END for each line + # END path handling + + if entries: + null_mode_entries = [ e for e in entries if e.mode == 0 ] + if null_mode_entries: + raise ValueError("At least one Entry has a null-mode - please use index.remove to remove files for clarity") + # END null mode should be remove + + # create objects if required, otherwise go with the existing shas + null_entries_indices = [ i for i,e in enumerate(entries) if e.sha == Object.NULL_HEX_SHA ] + if null_entries_indices: + hash_proc = self.repo.git.hash_object(w=True, stdin_paths=True, istream=subprocess.PIPE, as_process=True) + hash_proc.stdin.write('\n'.join(entries[i].path for i in null_entries_indices)) + obj_ids = self._flush_stdin_and_wait(hash_proc).splitlines() + assert len(obj_ids) == len(null_entries_indices), "git-hash-object did not produce all requested objects: want %i, got %i" % ( len(null_entries_indices), len(obj_ids) ) + + # update IndexEntries with new object id + for i,new_sha in zip(null_entries_indices, obj_ids): + e = entries[i] + new_entry = BaseIndexEntry((e.mode, new_sha, e.stage, e.path)) + entries[i] = new_entry + # END for each index + # END null_entry handling + + # feed all the data to stdin + update_index_proc = self.repo.git.update_index(index_info=True, istream=subprocess.PIPE, as_process=True, **kwargs) + update_index_proc.stdin.write('\n'.join(str(e) for e in entries)) + entries_added.extend(entries) + self._flush_stdin_and_wait(update_index_proc) + # END if there are base entries + + return entries_added @clear_cache @default_index @@ -768,6 +859,53 @@ class IndexFile(LazyMixin, diff.Diffable): fp.close() os.remove(tmp_file_path) + @classmethod + def _flush_stdin_and_wait(cls, proc): + proc.stdin.flush() + proc.stdin.close() + stdout = proc.stdout.read() + proc.wait() + return stdout + + @default_index + def checkout(self, paths=None, force=False, **kwargs): + """ + Checkout the given paths or all files from the version in the index. + + ``paths`` + If None, all paths in the index will be checked out. Otherwise an iterable + or single path of relative or absolute paths pointing to files is expected. + The command will ignore paths that do not exist. + + ``force`` + If True, existing files will be overwritten. If False, these will + be skipped. + + ``**kwargs`` + Additional arguments to be pasesd to git-checkout-index + + Returns + self + """ + args = ["--index"] + if force: + args.append("--force") + + if paths is None: + args.append("--all") + self.repo.git.checkout_index(*args, **kwargs) + else: + if not isinstance(paths, (tuple,list)): + paths = [paths] + + args.append("--stdin") + paths = [self._to_relative_path(p) for p in paths] + co_proc = self.repo.git.checkout_index(args, as_process=True, istream=subprocess.PIPE, **kwargs) + co_proc.stdin.write('\n'.join(paths)) + self._flush_stdin_and_wait(co_proc) + # END paths handling + return self + @clear_cache @default_index def reset(self, commit='HEAD', working_tree=False, paths=None, head=False, **kwargs): -- cgit v1.2.1 From 3cb5ba18ab1a875ef6b62c65342de476be47871b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 3 Nov 2009 16:35:33 +0100 Subject: object: renamed id attribute to sha as it in fact is always being rewritten as sha, even if the passed in id was a ref. This is done to assure objects are uniquely identified and will compare correctly --- lib/git/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index cc3f3a4e..705b1ae7 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -92,7 +92,7 @@ class BaseIndexEntry(tuple): Returns Fully equipped BaseIndexEntry at the given stage """ - return cls((blob.mode, blob.id, stage, blob.path)) + return cls((blob.mode, blob.sha, stage, blob.path)) class IndexEntry(BaseIndexEntry): @@ -165,7 +165,7 @@ class IndexEntry(BaseIndexEntry): Minimal entry resembling the given blob objecft """ time = struct.pack(">LL", 0, 0) - return IndexEntry((blob.mode, blob.id, 0, blob.path, time, time, 0, 0, 0, 0, blob.size)) + return IndexEntry((blob.mode, blob.sha, 0, blob.path, time, time, 0, 0, 0, 0, blob.size)) def clear_cache(func): -- cgit v1.2.1 From 4748e813980e1316aa364e0830a4dc082ff86eb0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 3 Nov 2009 21:43:17 +0100 Subject: added all new files to reference.rst and corrected the worst mistakes. There are still a few errors left that I cannot fix as it complains about whitespace in the end ... that is exactly what I hate restructured text for, its just a ... anyway. --- lib/git/index.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 705b1ae7..e368f531 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -44,11 +44,12 @@ class _TemporaryFileSwap(object): class BaseIndexEntry(tuple): """ - Small Brother of an index entry which can be created to describe changes + + Small Brother of an index entry which can be created to describe changes done to the index in which case plenty of additional information is not requried. - As the first 4 data members match exactly to the IndexEntry type, methods - expecting a BaseIndexEntry can also handle full IndexEntries even if they + As the first 4 data members match exactly to the IndexEntry type, methods + expecting a BaseIndexEntry can also handle full IndexEntries even if they use numeric indices for performance reasons. """ @@ -396,7 +397,7 @@ class IndexFile(LazyMixin, diff.Diffable): If 2 Trees are given, they will be merged into a new index using a two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' one. It behaves like a fast-forward. - If 3 Trees are given, a 3-way merge will be performed with the first tree + If 3 Trees are given, a 3-way merge will be performed with the first tree being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree, tree 3 is the 'other' one -- cgit v1.2.1