From a10aa36bc6a82bd50df6f3df7d6b7ce04a7070f1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 10:42:26 +0200 Subject: Renamed Index to IndexFile, adjusted tests, it will only operate on physical files, not on streams, as Indices are not streamed by any git command ( at least not in raw format ) --- lib/git/index.py | 150 ++++++++++++++++++------------------------------------- 1 file changed, 49 insertions(+), 101 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 4217c9a2..5f13790c 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -13,6 +13,7 @@ import mmap import objects import tempfile import os +import sys import stat import git.diff as diff @@ -34,7 +35,10 @@ class _TemporaryFileSwap(object): def __del__(self): if os.path.isfile(self.tmp_file_path): + if sys.platform == "win32" and os.path.exists(self.file_path): + os.remove(self.file_path) os.rename(self.tmp_file_path, self.file_path) + # END temp file exists class IndexEntry(tuple): @@ -142,7 +146,7 @@ class IndexEntry(tuple): return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) -class Index(LazyMixin, diff.Diffable): +class IndexFile(LazyMixin, diff.Diffable): """ Implements an Index that can be manipulated using a native implementation in order to save git command function calls wherever possible. @@ -155,14 +159,20 @@ class Index(LazyMixin, diff.Diffable): ``Entries`` The index contains an entries dict whose keys are tuples of type IndexEntry to facilitate access. + + As opposed to the Index type, the IndexFile represents the index on file level. + This can be considered an alternate, file-based implementation of the Index class + with less support for common functions. Use it for very special and custom index + handling. """ - __slots__ = ( "repo", "version", "entries", "_extension_data", "_is_default_index" ) + __slots__ = ( "repo", "version", "entries", "_extension_data", "_file_path" ) _VERSION = 2 # latest version we support S_IFGITLINK = 0160000 - def __init__(self, repo, stream = None): + def __init__(self, repo, file_path=None): """ - Initialize this Index instance, optionally from the given ``stream`` + Initialize this Index instance, optionally from the given ``file_path``. + If no file_path is given, we will be created from the current index file. If a stream is not given, the stream will be initialized from the current repository's index on demand. @@ -170,24 +180,27 @@ class Index(LazyMixin, diff.Diffable): self.repo = repo self.version = self._VERSION self._extension_data = '' - self._is_default_index = True - if stream is not None: - self._is_default_index = False - self._read_from_stream(stream) - # END read from stream immediatly - + self._file_path = file_path or self._index_path() def _set_cache_(self, attr): if attr == "entries": # read the current index - fp = open(self._index_path(), "r") + # try memory map for speed + fp = open(self._file_path, "r") + stream = fp try: - self._read_from_stream(fp) + stream = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) + except Exception: + pass + # END memory mapping + + try: + self._read_from_stream(stream) finally: fp.close() # END read from default index on demand else: - super(Index, self)._set_cache_(attr) + super(IndexFile, self)._set_cache_(attr) def _index_path(self): return os.path.join(self.repo.path, "index") @@ -197,12 +210,9 @@ class Index(LazyMixin, diff.Diffable): def path(self): """ Returns - Path to the index file we are representing or None if we are - a loose index that was read from a stream. + Path to the index file we are representing """ - if self._is_default_index: - return self._index_path() - return None + return self._file_path @classmethod def _read_entry(cls, stream): @@ -252,62 +262,8 @@ class Index(LazyMixin, diff.Diffable): # truncate the sha in the end as we will dynamically create it anyway self._extension_data = self._extension_data[:-20] - - @classmethod - def from_file(cls, repo, file_path): - """ - Returns - Index instance as recreated from the given stream. - - ``repo`` - Repository the index is related to - - ``file_pa `` - File path pointing to git index file - - Note - Reading is based on the dulwich project. - """ - fp = open(file_path, "r") - - # try memory map for speed - stream = fp - try: - stream = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) - except Exception: - pass - # END memory mapping - - try: - return cls(repo, stream) - finally: - fp.close() - - @classmethod - def to_file(cls, index, file_path): - """ - Write the index data to the given file path. - - ``index`` - Index you wish to write. - - ``file_path`` - Path at which to write the index data. Please note that missing directories - will lead to an exception to be thrown. - - Raise - IOError if the file could not be written - """ - fp = open(file_path, "w") - try: - return index.write(fp) - finally: - fp.close() - # END exception handling - - @classmethod def _write_cache_entry(cls, stream, entry): """ @@ -326,14 +282,14 @@ class Index(LazyMixin, diff.Diffable): real_size = ((stream.tell() - beginoffset + 8) & ~7) stream.write("\0" * ((beginoffset + real_size) - stream.tell())) - def write(self, stream=None): + def write(self, file_path = None): """ - Write the current state to the given stream or to the default repository - index. + Write the current state to our file path or to the given one - ``stream`` - File-like object or None. - If None, the default repository index will be overwritten. + ``file_path`` + If None, we will write to our stored file path from which we have + been initialized. Otherwise we write to the given file path. + Please note that this will not change the file_path of this index. Returns self @@ -341,12 +297,8 @@ class Index(LazyMixin, diff.Diffable): Note Index writing based on the dulwich implementation """ - write_op = None - if stream is None: - write_op = ConcurrentWriteOperation(self._index_path()) - stream = write_op._begin_writing() - # stream = open(self._index_path() - # END stream handling + write_op = ConcurrentWriteOperation(file_path or self._file_path) + stream = write_op._begin_writing() stream = SHA1Writer(stream) @@ -366,9 +318,7 @@ class Index(LazyMixin, diff.Diffable): # write the sha over the content stream.write_sha() - - if write_op is not None: - write_op._end_writing() + write_op._end_writing() @classmethod @@ -394,6 +344,11 @@ class Index(LazyMixin, diff.Diffable): ``**kwargs`` Additional arguments passed to git-read-tree + Returns + New IndexFile instance. It will point to a temporary index location which + does not exist anymore. If you intend to write such a merged Index, supply + an alternate file_path to its 'write' method. + Note: In the three-way merge case, --aggressive will be specified to automatically resolve more cases in a commonly correct manner. Specify trivial=True as kwarg @@ -428,7 +383,8 @@ class Index(LazyMixin, diff.Diffable): index_handler = _TemporaryFileSwap(os.path.join(repo.path, 'index')) try: repo.git.read_tree(*arg_list, **kwargs) - index = cls.from_file(repo, tmp_index) + index = cls(repo, tmp_index) + index.entries # force it to read the file finally: if os.path.exists(tmp_index): os.remove(tmp_index) @@ -539,20 +495,11 @@ class Index(LazyMixin, diff.Diffable): index_path = self._index_path() tmp_index_mover = _TemporaryFileSwap(index_path) - self.to_file(self, index_path) + self.write(index_path) + tree_sha = self.repo.git.write_tree() - try: - tree_sha = self.repo.git.write_tree() - finally: - # remove our index file so that the original index can move back into place - # On linux it will silently overwrite, on windows it won't - if os.path.isfile(index_path): - os.remove(index_path) - # END remove our own index file beforehand - # END write tree handling return Tree(self.repo, tree_sha, 0, '') - def _process_diff_args(self, args): try: args.pop(args.index(self)) @@ -572,7 +519,8 @@ class Index(LazyMixin, diff.Diffable): Will only work with indices that represent the default git index as they have not been initialized with a stream. """ - if not self._is_default_index: + # perhaps we shouldn't diff these at all, or we swap them in place first + if self._file_path != self._index_path(): raise AssertionError( "Cannot diff custom indices as they do not represent the default git index" ) # index against index is always empty @@ -598,5 +546,5 @@ class Index(LazyMixin, diff.Diffable): raise ValueError( "other must be None, Diffable.Index, a Tree or Commit, was %r" % other ) # diff against working copy - can be handled by superclass natively - return super(Index, self).diff(other, paths, create_patch, **kwargs) + return super(IndexFile, self).diff(other, paths, create_patch, **kwargs) -- cgit v1.2.1 From 3d3a24b22d340c62fafc0e75a349c0ffe34d99d7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 11:07:04 +0200 Subject: Added repo.index property including simple test, and additional ideas in the TODO list --- lib/git/index.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 5f13790c..5452708d 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -289,7 +289,8 @@ class IndexFile(LazyMixin, diff.Diffable): ``file_path`` If None, we will write to our stored file path from which we have been initialized. Otherwise we write to the given file path. - Please note that this will not change the file_path of this index. + Please note that this will change the file_path of this index to + the one you gave. Returns self @@ -319,7 +320,10 @@ class IndexFile(LazyMixin, diff.Diffable): # write the sha over the content stream.write_sha() write_op._end_writing() - + + # make sure we represent what we have written + if file_path is not None: + self._file_path = file_path @classmethod def from_tree(cls, repo, *treeish, **kwargs): -- cgit v1.2.1 From a7a4388eeaa4b6b94192dce67257a34c4a6cbd26 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 12:14:22 +0200 Subject: Added frame for IndexFile add/remove/commit methods and respective test markers --- lib/git/index.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 5452708d..9fd1fad2 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -146,6 +146,22 @@ class IndexEntry(tuple): return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) +def default_index(func): + """ + Decorator assuring the wrapped method may only run if we are the default + repository index. This is as we rely on git commands that operate + on that index only. + """ + def check_default_index(self, *args, **kwargs): + if self._file_path != self._index_path(): + raise AssertionError( "Cannot call %r on indices that do not represent the default git index" % func.__name__ ) + return func(self, *args, **kwargs) + # END wrpaper method + + check_default_index.__name__ = func.__name__ + return check_default_index + + class IndexFile(LazyMixin, diff.Diffable): """ Implements an Index that can be manipulated using a native implementation in @@ -487,6 +503,22 @@ class IndexFile(LazyMixin, diff.Diffable): # END for each blob return self + + def update(self): + """ + Reread the contents of our index file, discarding all cached information + we might have. + + Note: + This is a possibly dangerious operations as it will discard your changes + to index.endtries + + Returns + self + """ + del(self.entries) + self.entries + return self def write_tree(self): """ @@ -512,6 +544,88 @@ class IndexFile(LazyMixin, diff.Diffable): # END remove self return args + @default_index + def add(self, items, **kwargs): + """ + Add files from the working copy, specific blobs or IndexEntries + to the index. + + TODO: Its important to specify a way to add symlinks directly, even + on systems that do not support it, like ... erm ... windows. + + ``**kwargs`` + Additional keyword arguments to be passed to git-update-index + + Returns + List(IndexEntries) representing the entries just added + """ + raise NotImplementedError("todo") + + @default_index + def remove(self, items, affect_working_tree=False, **kwargs): + """ + Remove the given file_paths or blobs from the index and optionally from + the working tree as well. + + ``items`` + TODO + + ``affect_working_tree`` + If True, the entry will also be removed from the working tree, physically + removing the respective file. This may fail if there are uncommited changes + in it. + + ``**kwargs`` + Additional keyword arguments to be passed to git-update-index + + Returns + self + """ + raise NotImplementedError("todo") + return self + + @default_index + def commit(self, message=None, parent_commits=None, **kwargs): + """ + Commit the current index, creating a commit object. + + ``message`` + Commit message + + ``parent_commits`` + Optional Commit objects to use as parents for the new commit. + If None or empty, the current head commit will be the parent of the + new commit object + + ``**kwargs`` + Additional keyword arguments passed to git-commit + + Returns + Commit object representing the new commit + """ + raise NotImplementedError("todo") + + @default_index + def reset(self, commit='HEAD', working_tree=False, **kwargs): + """ + Reset the index to reflect the tree at the given commit. This will not + adjust our HEAD reference as opposed to HEAD.reset. + + ``commit`` + Revision, Reference or Commit specifying the commit we should represent. + If you want to specify a tree only, use IndexFile.from_tree and overwrite + the default index. + + ``working_tree`` + If True, the files in the working tree will reflect the changed index. + If False, the working tree will not be touched + + ``**kwargs`` + Additional keyword arguments passed to git-reset + """ + raise NotImplementedError("todo") + + @default_index def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): """ Diff this index against the working copy or a Tree or Commit object @@ -523,10 +637,6 @@ class IndexFile(LazyMixin, diff.Diffable): Will only work with indices that represent the default git index as they have not been initialized with a stream. """ - # perhaps we shouldn't diff these at all, or we swap them in place first - if self._file_path != self._index_path(): - raise AssertionError( "Cannot diff custom indices as they do not represent the default git index" ) - # index against index is always empty if other is self.Index: return diff.DiffIndex() -- cgit v1.2.1 From 81d8788d40867f4e5cf3016ef219473951a7f6ed Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 17:19:36 +0200 Subject: IndexEntry is now based on a 'minimal' version that is suitable to be fed into UpdateIndex. The Inode and device information is only needed to quickly compare the index against the working tree for changes, hence it should not be that dominant in the API either. More changes to come --- lib/git/index.py | 121 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 52 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 9fd1fad2..754c9f15 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -41,7 +41,57 @@ class _TemporaryFileSwap(object): # END temp file exists -class IndexEntry(tuple): +class BaseIndexEntry(tuple): + """ + Small Brother of an index entry which can be created to describe changes + done to the index in which case plenty of additional information is not requried. + + As the first 4 data members match exactly to the IndexEntry type, methods + expecting a BaseIndexEntry can also handle full IndexEntries even if they + use numeric indices for performance reasons. + """ + + @property + def mode(self): + """ + File Mode, compatible to stat module constants + """ + return self[0] + + @property + def sha(self): + """ + hex sha of the blob + """ + return self[1] + + @property + def stage(self): + """ + Stage of the entry, either: + 0 = default stage + 1 = stage before a merge or common ancestor entry in case of a 3 way merge + 2 = stage of entries from the 'left' side of the merge + 3 = stage of entries from the right side of the merge + Note: + For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html + """ + return self[2] + + @property + def path(self): + return self[3] + + @classmethod + def from_blob(cls, blob, stage = 0): + """ + Returns + Fully equipped BaseIndexEntry at the given stage + """ + return cls((blob.mode, blob.id, stage, blob.path)) + + +class IndexEntry(BaseIndexEntry): """ Allows convenient access to IndexEntry data without completely unpacking it. @@ -57,49 +107,42 @@ class IndexEntry(tuple): Tuple(int_time_seconds_since_epoch, int_nano_seconds) of the file's creation time """ - return struct.unpack(">LL", self[0]) + return struct.unpack(">LL", self[4]) @property def mtime(self): """ See ctime property, but returns modification time """ - return struct.unpack(">LL", self[1]) + return struct.unpack(">LL", self[5]) @property def dev(self): """ Device ID """ - return self[2] + return self[6] @property def inode(self): """ Inode ID """ - return self[3] - - @property - def mode(self): - """ - File Mode, compatible to stat module constants - """ - return self[4] - + return self[7] + @property def uid(self): """ User ID """ - return self[5] + return self[8] @property def gid(self): """ Group ID """ - return self[6] + return self[9] @property def size(self): @@ -109,33 +152,8 @@ class IndexEntry(tuple): Note Will be 0 if the stage is not 0 ( hence it is an unmerged entry ) """ - return self[7] - - @property - def sha(self): - """ - hex sha of the blob - """ - return self[8] - - @property - def stage(self): - """ - Stage of the entry, either: - 0 = default stage - 1 = stage before a merge or common ancestor entry in case of a 3 way merge - 2 = stage of entries from the 'left' side of the merge - 3 = stage of entries from the right side of the merge - Note: - For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html - """ - return self[9] - - @property - def path(self): return self[10] - @classmethod def from_blob(cls, blob): """ @@ -143,7 +161,7 @@ class IndexEntry(tuple): Minimal entry resembling the given blob objecft """ time = struct.pack(">LL", 0, 0) - return IndexEntry((time, time, 0, 0, blob.mode, 0, 0, blob.size, blob.id, 0, blob.path)) + return IndexEntry((blob.mode, blob.id, 0, blob.path, time, time, 0, 0, 0, 0, blob.size)) def default_index(func): @@ -243,8 +261,7 @@ class IndexFile(LazyMixin, diff.Diffable): real_size = ((stream.tell() - beginoffset + 8) & ~7) data = stream.read((beginoffset + real_size) - stream.tell()) - return IndexEntry((ctime, mtime, dev, ino, mode, uid, gid, size, - binascii.hexlify(sha), flags >> 12, path)) + return IndexEntry((mode, binascii.hexlify(sha), flags >> 12, path, ctime, mtime, dev, ino, uid, gid, size)) @classmethod def _read_header(cls, stream): @@ -286,14 +303,14 @@ class IndexFile(LazyMixin, diff.Diffable): Write an IndexEntry to a stream """ beginoffset = stream.tell() - stream.write(entry[0]) # ctime - stream.write(entry[1]) # mtime - path = entry[10] + stream.write(entry[4]) # ctime + stream.write(entry[5]) # mtime + path = entry[3] plen = len(path) & 0x0fff # path length - assert plen == len(path), "Path %s too long to fit into index" % entry[10] - flags = plen | (entry[9] << 12)# stage and path length are 2 byte flags - stream.write(struct.pack(">LLLLLL20sH", entry[2], entry[3], entry[4], - entry[5], entry[6], entry[7], binascii.unhexlify(entry[8]), flags)) + assert plen == len(path), "Path %s too long to fit into index" % entry[3] + flags = plen | (entry[2] << 12)# stage and path length are 2 byte flags + stream.write(struct.pack(">LLLLLL20sH", entry[6], entry[7], entry[0], + entry[8], entry[9], entry[10], binascii.unhexlify(entry[1]), flags)) stream.write(path) real_size = ((stream.tell() - beginoffset + 8) & ~7) stream.write("\0" * ((beginoffset + real_size) - stream.tell())) @@ -325,7 +342,7 @@ class IndexFile(LazyMixin, diff.Diffable): # body entries_sorted = self.entries.values() - entries_sorted.sort(key=lambda e: (e[10], e[9])) # use path/stage as sort key + entries_sorted.sort(key=lambda e: (e[3], e[2])) # use path/stage as sort key for entry in entries_sorted: self._write_cache_entry(stream, entry) # END for each entry @@ -623,7 +640,7 @@ class IndexFile(LazyMixin, diff.Diffable): ``**kwargs`` Additional keyword arguments passed to git-reset """ - raise NotImplementedError("todo") + raise NotImplementedError("todo: use git-read-tree if there is no working tree to update") @default_index def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): -- cgit v1.2.1 From 13a26d4f9c22695033040dfcd8c76fd94187035b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 17:55:33 +0200 Subject: Implemented index.reset method including test --- lib/git/index.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 754c9f15..111dc8d2 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -528,13 +528,13 @@ class IndexFile(LazyMixin, diff.Diffable): Note: This is a possibly dangerious operations as it will discard your changes - to index.endtries + to index.entries Returns self """ del(self.entries) - self.entries + # allows to lazily reread on demand return self def write_tree(self): @@ -623,7 +623,7 @@ class IndexFile(LazyMixin, diff.Diffable): raise NotImplementedError("todo") @default_index - def reset(self, commit='HEAD', working_tree=False, **kwargs): + def reset(self, commit='HEAD', working_tree=False, paths=None, **kwargs): """ Reset the index to reflect the tree at the given commit. This will not adjust our HEAD reference as opposed to HEAD.reset. @@ -636,11 +636,23 @@ class IndexFile(LazyMixin, diff.Diffable): ``working_tree`` If True, the files in the working tree will reflect the changed index. If False, the working tree will not be touched + Please note that changes to the working copy will be discarded without + warning ! ``**kwargs`` Additional keyword arguments passed to git-reset + + Returns + self """ - raise NotImplementedError("todo: use git-read-tree if there is no working tree to update") + head = self.repo.head + prev_commit = head.commit + + # reset to get the tree/working copy + head.reset(commit, index=True, working_tree=working_tree, paths=paths, **kwargs) + # put the head back + head.reset(prev_commit, index=False, working_tree=False) + return self @default_index def diff(self, other=diff.Diffable.Index, paths=None, create_patch=False, **kwargs): -- cgit v1.2.1 From 0cd09bd306486028f5442c56ef2e947355a06282 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Oct 2009 21:49:13 +0200 Subject: index.remove implemented including throrough test --- lib/git/index.py | 152 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 131 insertions(+), 21 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 111dc8d2..65e658b0 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -164,6 +164,26 @@ class IndexEntry(BaseIndexEntry): return IndexEntry((blob.mode, blob.id, 0, blob.path, time, time, 0, 0, 0, 0, blob.size)) +def clear_cache(func): + """ + Decorator for functions that alter the index using the git command. This would + invalidate our possibly existing entries dictionary which is why it must be + deleted to allow it to be lazily reread later. + + Note + This decorator will not be required once all functions are implemented + natively which in fact is possible, but probably not feasible performance wise. + """ + def clear_cache_if_not_raised(self, *args, **kwargs): + rval = func(self, *args, **kwargs) + del(self.entries) + return rval + + # END wrapper method + clear_cache_if_not_raised.__name__ = func.__name__ + return clear_cache_if_not_raised + + def default_index(func): """ Decorator assuring the wrapped method may only run if we are the default @@ -194,10 +214,8 @@ class IndexFile(LazyMixin, diff.Diffable): The index contains an entries dict whose keys are tuples of type IndexEntry to facilitate access. - As opposed to the Index type, the IndexFile represents the index on file level. - This can be considered an alternate, file-based implementation of the Index class - with less support for common functions. Use it for very special and custom index - handling. + You may only read the entries dict or manipulate it through designated methods. + Otherwise changes to it will be lost when changing the index using its methods. """ __slots__ = ( "repo", "version", "entries", "_extension_data", "_file_path" ) _VERSION = 2 # latest version we support @@ -500,6 +518,10 @@ class IndexFile(LazyMixin, diff.Diffable): Returns: self + + Note + You will have to write the index manually once you are done, i.e. + index.resolve_blobs(blobs).write() """ for blob in iter_blobs: stage_null_key = (blob.path, 0) @@ -561,45 +583,132 @@ class IndexFile(LazyMixin, diff.Diffable): # END remove self return args + + def _to_relative_path(self, path): + """ + Return + Version of path relative to our git directory or raise ValueError + if it is not within our git direcotory + """ + if not os.path.isabs(path): + return path + relative_path = path.replace(self.repo.git.git_dir+"/", "") + if relative_path == path: + raise ValueError("Absolute path %r is not in git repository at %r" % (path,self.repo.git.git_dir)) + return relative_path + + @clear_cache @default_index def add(self, items, **kwargs): """ - Add files from the working copy, specific blobs or IndexEntries - to the index. - - TODO: Its important to specify a way to add symlinks directly, even - on systems that do not support it, like ... erm ... windows. + Add files from the working tree, specific blobs or BaseIndexEntries + to the index. The underlying index file will be written immediately, hence + you should provide as many items as possible to minimize the amounts of writes + ``items`` + Multiple types of items are supported, types can be mixed within one call. + Different types imply a different handling. File paths may generally be + relative or absolute. + + - path string + strings denote a relative or absolute path into the repository pointing to + an existing file, i.e. CHANGES, lib/myfile.ext, /home/gitrepo/lib/myfile.ext. + + Paths provided like this must exist. When added, they will be written + into the object database. + + This equals a straight git-add. + + They are added at stage 0 + + - Blob object + Blobs are added as they are assuming a valid mode is set. + The file they refer to may or may not exist in the file system + + If their sha is null ( 40*0 ), their path must exist in the file system + as an object will be created from the data at the path.The handling + now very much equals the way string paths are processed, except that + the mode you have set will be kept. This allows you to create symlinks + by settings the mode respectively and writing the target of the symlink + directly into the file. This equals a default Linux-Symlink which + is not dereferenced automatically, except that it can be created on + filesystems not supporting it as well. + + They are added at stage 0 + + - BaseIndexEntry or type + Handling equals the one of Blob objects, but the stage may be + explicitly set. + ``**kwargs`` - Additional keyword arguments to be passed to git-update-index + Additional keyword arguments to be passed to git-update-index, such + as index_only. Returns - List(IndexEntries) representing the entries just added + List(BaseIndexEntries) representing the entries just actually added. """ raise NotImplementedError("todo") + @clear_cache @default_index - def remove(self, items, affect_working_tree=False, **kwargs): + def remove(self, items, working_tree=False, **kwargs): """ - Remove the given file_paths or blobs from the index and optionally from + Remove the given items from the index and optionally from the working tree as well. ``items`` - TODO + Multiple types of items are supported which may be be freely mixed. + + - path string + Remove the given path at all stages. If it is a directory, you must + specify the r=True keyword argument to remove all file entries + below it. If absolute paths are given, they will be converted + to a path relative to the git repository directory containing + the working tree + + The path string may include globs, such as *.c. + + - Blob object + Only the path portion is used in this case. + + - BaseIndexEntry or compatible type + The only relevant information here Yis the path. The stage is ignored. - ``affect_working_tree`` + ``working_tree`` If True, the entry will also be removed from the working tree, physically removing the respective file. This may fail if there are uncommited changes in it. ``**kwargs`` - Additional keyword arguments to be passed to git-update-index + Additional keyword arguments to be passed to git-rm, such + as 'r' to allow recurive removal of Returns - self - """ - raise NotImplementedError("todo") - return self + List(path_string, ...) list of paths that have been removed effectively. + This is interesting to know in case you have provided a directory or + globs. Paths are relative to the + """ + args = list() + if not working_tree: + args.append("--cached") + args.append("--") + + # preprocess paths + paths = list() + for item in items: + if isinstance(item, (BaseIndexEntry,Blob)): + paths.append(self._to_relative_path(item.path)) + elif isinstance(item, basestring): + paths.append(self._to_relative_path(item)) + else: + raise TypeError("Invalid item type: %r" % item) + # END for each item + + removed_paths = self.repo.git.rm(args, paths, **kwargs).splitlines() + + # process output to gain proper paths + # rm 'path' + return [ p[4:-1] for p in removed_paths ] @default_index def commit(self, message=None, parent_commits=None, **kwargs): @@ -621,7 +730,8 @@ class IndexFile(LazyMixin, diff.Diffable): Commit object representing the new commit """ raise NotImplementedError("todo") - + + @clear_cache @default_index def reset(self, commit='HEAD', working_tree=False, paths=None, **kwargs): """ -- cgit v1.2.1 From f9cec00938d9059882bb8eabdaf2f775943e00e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Oct 2009 00:08:33 +0200 Subject: index.commit: implemented initial version, but in fact some more changes are required to have a nice API. Tests are not yet fully done either --- lib/git/index.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 65e658b0..86bdfd39 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -17,7 +17,7 @@ import sys import stat import git.diff as diff -from git.objects import Blob, Tree, Object +from git.objects import Blob, Tree, Object, Commit from git.utils import SHA1Writer, LazyMixin, ConcurrentWriteOperation @@ -711,25 +711,51 @@ class IndexFile(LazyMixin, diff.Diffable): return [ p[4:-1] for p in removed_paths ] @default_index - def commit(self, message=None, parent_commits=None, **kwargs): + def commit(self, message, parent_commits=None): """ Commit the current index, creating a commit object. ``message`` - Commit message + Commit message. It may be an empty string if no message is provided. + It will be converted to a string in any case. ``parent_commits`` - Optional Commit objects to use as parents for the new commit. - If None or empty, the current head commit will be the parent of the + Optional Commit objects to use as parents for the new commit. + If empty list, the commit will have no parents at all and become + a root commit. + If None , the current head commit will be the parent of the new commit object - ``**kwargs`` - Additional keyword arguments passed to git-commit - Returns Commit object representing the new commit + + Note: + Additional information about hte committer and Author are taken from the + environment or from the git configuration, see git-commit-tree for + more information """ - raise NotImplementedError("todo") + parents = parent_commits + if parent_commits is None: + parent_commits = [ self.repo.head.commit ] + + parent_args = [ ("-p", str(commit)) for commit in parent_commits ] + + # create message stream + tmp_file_path = tempfile.mktemp() + fp = open(tmp_file_path,"w") + fp.write(str(message)) + fp.close() + fp = open(tmp_file_path,"r") + fp.seek(0) + + try: + # write the current index as tree + tree_sha = self.repo.git.write_tree() + commit_sha = self.repo.git.commit_tree(tree_sha, parent_args, istream=fp) + return Commit(self.repo, commit_sha) + finally: + fp.close() + os.remove(tmp_file_path) @clear_cache @default_index -- cgit v1.2.1 From 5ba2aef8f59009756567a53daaf918afa851c304 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Oct 2009 11:25:45 +0100 Subject: added head kwarg to reset and commit method, allowing to automatically change the head to the given commit, which makes the methods more versatile --- lib/git/index.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 86bdfd39..89e716d4 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -711,7 +711,7 @@ class IndexFile(LazyMixin, diff.Diffable): return [ p[4:-1] for p in removed_paths ] @default_index - def commit(self, message, parent_commits=None): + def commit(self, message, parent_commits=None, head=True): """ Commit the current index, creating a commit object. @@ -726,6 +726,11 @@ class IndexFile(LazyMixin, diff.Diffable): If None , the current head commit will be the parent of the new commit object + ``head`` + If True, the HEAD will be advanced to the new commit automatically. + Else the HEAD will remain pointing on the previous commit. This could + lead to undesired results when diffing files. + Returns Commit object representing the new commit @@ -752,17 +757,23 @@ class IndexFile(LazyMixin, diff.Diffable): # write the current index as tree tree_sha = self.repo.git.write_tree() commit_sha = self.repo.git.commit_tree(tree_sha, parent_args, istream=fp) - return Commit(self.repo, commit_sha) + new_commit = Commit(self.repo, commit_sha) + + if head: + self.repo.head.commit = new_commit + # END advance head handling + + return new_commit finally: fp.close() os.remove(tmp_file_path) - + @clear_cache @default_index - def reset(self, commit='HEAD', working_tree=False, paths=None, **kwargs): + def reset(self, commit='HEAD', working_tree=False, paths=None, head=False, **kwargs): """ Reset the index to reflect the tree at the given commit. This will not - adjust our HEAD reference as opposed to HEAD.reset. + adjust our HEAD reference as opposed to HEAD.reset by default. ``commit`` Revision, Reference or Commit specifying the commit we should represent. @@ -775,19 +786,27 @@ class IndexFile(LazyMixin, diff.Diffable): Please note that changes to the working copy will be discarded without warning ! + ``head`` + If True, the head will be set to the given commit. This is False by default, + but if True, this method behaves like HEAD.reset. + ``**kwargs`` Additional keyword arguments passed to git-reset Returns self """ - head = self.repo.head - prev_commit = head.commit + cur_head = self.repo.head + prev_commit = cur_head.commit # reset to get the tree/working copy - head.reset(commit, index=True, working_tree=working_tree, paths=paths, **kwargs) - # put the head back - head.reset(prev_commit, index=False, working_tree=False) + cur_head.reset(commit, index=True, working_tree=working_tree, paths=paths, **kwargs) + + # put the head back, possibly + if not head: + cur_head.reset(prev_commit, index=False, working_tree=False) + # END reset head + return self @default_index -- cgit v1.2.1 From 0ef1f89abe5b2334705ee8f1a6da231b0b6c9a50 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Oct 2009 22:37:48 +0100 Subject: index.add: Finished implemenation including through tests index.checkout: added simple method allowing to checkout files from the index, including simple test --- lib/git/index.py | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 142 insertions(+), 4 deletions(-) (limited to 'lib/git/index.py') diff --git a/lib/git/index.py b/lib/git/index.py index 89e716d4..cc3f3a4e 100644 --- a/lib/git/index.py +++ b/lib/git/index.py @@ -15,6 +15,7 @@ import tempfile import os import sys import stat +import subprocess import git.diff as diff from git.objects import Blob, Tree, Object, Commit @@ -51,6 +52,9 @@ class BaseIndexEntry(tuple): use numeric indices for performance reasons. """ + def __str__(self): + return "%o %s %i\t%s\n" % (self.mode, self.sha, self.stage, self.path) + @property def mode(self): """ @@ -466,6 +470,12 @@ class IndexFile(LazyMixin, diff.Diffable): ret |= (index_mode & 0111) return ret + @classmethod + def _tree_mode_to_index_mode(cls, tree_mode): + """ + Convert a tree mode to index mode as good as possible + """ + def iter_blobs(self, predicate = lambda t: True): """ Returns @@ -597,9 +607,29 @@ class IndexFile(LazyMixin, diff.Diffable): raise ValueError("Absolute path %r is not in git repository at %r" % (path,self.repo.git.git_dir)) return relative_path + def _preprocess_add_items(self, items): + """ + Split the items into two lists of path strings and BaseEntries. + """ + paths = list() + entries = list() + + for item in items: + if isinstance(item, basestring): + paths.append(self._to_relative_path(item)) + elif isinstance(item, Blob): + entries.append(BaseIndexEntry.from_blob(item)) + elif isinstance(item, BaseIndexEntry): + entries.append(item) + else: + raise TypeError("Invalid Type: %r" % item) + # END for each item + return (paths, entries) + + @clear_cache @default_index - def add(self, items, **kwargs): + def add(self, items, force=True, **kwargs): """ Add files from the working tree, specific blobs or BaseIndexEntries to the index. The underlying index file will be written immediately, hence @@ -612,18 +642,23 @@ class IndexFile(LazyMixin, diff.Diffable): - path string strings denote a relative or absolute path into the repository pointing to - an existing file, i.e. CHANGES, lib/myfile.ext, /home/gitrepo/lib/myfile.ext. + an existing file, i.e. CHANGES, lib/myfile.ext, '/home/gitrepo/lib/myfile.ext'. Paths provided like this must exist. When added, they will be written into the object database. + PathStrings may contain globs, such as 'lib/__init__*' or can be directories + like 'lib', the latter ones will add all the files within the dirctory and + subdirectories. + This equals a straight git-add. They are added at stage 0 - Blob object Blobs are added as they are assuming a valid mode is set. - The file they refer to may or may not exist in the file system + The file they refer to may or may not exist in the file system, but + must be a path relative to our repository. If their sha is null ( 40*0 ), their path must exist in the file system as an object will be created from the data at the path.The handling @@ -634,12 +669,21 @@ class IndexFile(LazyMixin, diff.Diffable): is not dereferenced automatically, except that it can be created on filesystems not supporting it as well. + Please note that globs or directories are not allowed in Blob objects. + They are added at stage 0 - BaseIndexEntry or type Handling equals the one of Blob objects, but the stage may be explicitly set. + ``force`` + If True, otherwise ignored or excluded files will be + added anyway. + As opposed to the git-add command, we enable this flag by default + as the API user usually wants the item to be added even though + they might be excluded. + ``**kwargs`` Additional keyword arguments to be passed to git-update-index, such as index_only. @@ -647,7 +691,54 @@ class IndexFile(LazyMixin, diff.Diffable): Returns List(BaseIndexEntries) representing the entries just actually added. """ - raise NotImplementedError("todo") + # sort the entries into strings and Entries, Blobs are converted to entries + # automatically + # paths can be git-added, for everything else we use git-update-index + entries_added = list() + paths, entries = self._preprocess_add_items(items) + + if paths: + git_add_output = self.repo.git.add(paths, v=True) + # force rereading our entries + del(self.entries) + for line in git_add_output.splitlines(): + # line contains: + # add '' + added_file = line[5:-1] + entries_added.append(self.entries[(added_file,0)]) + # END for each line + # END path handling + + if entries: + null_mode_entries = [ e for e in entries if e.mode == 0 ] + if null_mode_entries: + raise ValueError("At least one Entry has a null-mode - please use index.remove to remove files for clarity") + # END null mode should be remove + + # create objects if required, otherwise go with the existing shas + null_entries_indices = [ i for i,e in enumerate(entries) if e.sha == Object.NULL_HEX_SHA ] + if null_entries_indices: + hash_proc = self.repo.git.hash_object(w=True, stdin_paths=True, istream=subprocess.PIPE, as_process=True) + hash_proc.stdin.write('\n'.join(entries[i].path for i in null_entries_indices)) + obj_ids = self._flush_stdin_and_wait(hash_proc).splitlines() + assert len(obj_ids) == len(null_entries_indices), "git-hash-object did not produce all requested objects: want %i, got %i" % ( len(null_entries_indices), len(obj_ids) ) + + # update IndexEntries with new object id + for i,new_sha in zip(null_entries_indices, obj_ids): + e = entries[i] + new_entry = BaseIndexEntry((e.mode, new_sha, e.stage, e.path)) + entries[i] = new_entry + # END for each index + # END null_entry handling + + # feed all the data to stdin + update_index_proc = self.repo.git.update_index(index_info=True, istream=subprocess.PIPE, as_process=True, **kwargs) + update_index_proc.stdin.write('\n'.join(str(e) for e in entries)) + entries_added.extend(entries) + self._flush_stdin_and_wait(update_index_proc) + # END if there are base entries + + return entries_added @clear_cache @default_index @@ -768,6 +859,53 @@ class IndexFile(LazyMixin, diff.Diffable): fp.close() os.remove(tmp_file_path) + @classmethod + def _flush_stdin_and_wait(cls, proc): + proc.stdin.flush() + proc.stdin.close() + stdout = proc.stdout.read() + proc.wait() + return stdout + + @default_index + def checkout(self, paths=None, force=False, **kwargs): + """ + Checkout the given paths or all files from the version in the index. + + ``paths`` + If None, all paths in the index will be checked out. Otherwise an iterable + or single path of relative or absolute paths pointing to files is expected. + The command will ignore paths that do not exist. + + ``force`` + If True, existing files will be overwritten. If False, these will + be skipped. + + ``**kwargs`` + Additional arguments to be pasesd to git-checkout-index + + Returns + self + """ + args = ["--index"] + if force: + args.append("--force") + + if paths is None: + args.append("--all") + self.repo.git.checkout_index(*args, **kwargs) + else: + if not isinstance(paths, (tuple,list)): + paths = [paths] + + args.append("--stdin") + paths = [self._to_relative_path(p) for p in paths] + co_proc = self.repo.git.checkout_index(args, as_process=True, istream=subprocess.PIPE, **kwargs) + co_proc.stdin.write('\n'.join(paths)) + self._flush_stdin_and_wait(co_proc) + # END paths handling + return self + @clear_cache @default_index def reset(self, commit='HEAD', working_tree=False, paths=None, head=False, **kwargs): -- cgit v1.2.1