From 989671780551b7587d57e1d7cb5eb1002ade75b4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 19 Oct 2009 23:44:18 +0200 Subject: Implemneted IterableLists for refs, commits and remote objects including simple tests --- lib/git/utils.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index f84c247d..3e7e8e75 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -56,12 +56,46 @@ class LazyMixin(object): pass +class IterableList(list): + """ + List of iterable objects allowing to query an object by id or by named index:: + + heads = repo.heads + heads.master + heads['master'] + heads[0] + """ + __slots__ = '_id_attr' + + def __new__(cls, id_attr): + return super(IterableList,cls).__new__(cls) + + def __init__(self, id_attr): + self._id_attr = id_attr + + def __getattr__(self, attr): + for item in self: + if getattr(item, self._id_attr) == attr: + return item + # END for each item + return list.__getattribute__(self, attr) + + def __getitem__(self, index): + if isinstance(index, int): + return list.__getitem__(self,index) + + try: + return getattr(self, index) + except AttributeError: + raise IndexError( "No item found with id %r" % index ) + class Iterable(object): """ Defines an interface for iterable items which is to assure a uniform way to retrieve and iterate items within the git repository """ __slots__ = tuple() + _id_attribute_ = "attribute that most suitably identifies your instance" @classmethod def list_items(cls, repo, *args, **kwargs): @@ -75,7 +109,10 @@ class Iterable(object): Returns: list(Item,...) list of item instances """ - return list(cls.iter_items(repo, *args, **kwargs)) + #return list(cls.iter_items(repo, *args, **kwargs)) + out_list = IterableList( cls._id_attribute_ ) + out_list.extend(cls.iter_items(repo, *args, **kwargs)) + return out_list @classmethod -- cgit v1.2.1 From aa5e366889103172a9829730de1ba26d3dcbc01b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 20 Oct 2009 10:11:16 +0200 Subject: Moved specialized methods like dashify, touch and is_git_dir to module to the respective modules that use them fixed repo.daemon_export which did not work anymore due to incorrect touch implementation and wrong property names --- lib/git/utils.py | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index 3e7e8e75..2aa8d33e 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -6,26 +6,6 @@ import os -def dashify(string): - return string.replace('_', '-') - -def touch(filename): - os.utime(filename) - -def is_git_dir(d): - """ This is taken from the git setup.c:is_git_directory - function.""" - - if os.path.isdir(d) and \ - os.path.isdir(os.path.join(d, 'objects')) and \ - os.path.isdir(os.path.join(d, 'refs')): - headref = os.path.join(d, 'HEAD') - return os.path.isfile(headref) or \ - (os.path.islink(headref) and - os.readlink(headref).startswith('refs')) - return False - - class LazyMixin(object): """ Base class providing an interface to lazily retrieve attribute values upon -- cgit v1.2.1 From 30d822a468dc909aac5c83d078a59bfc85fc27aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 10:15:47 +0200 Subject: index writing now creates a sha on the content making it possible to write valid indices after manually removing or altering entriesgst --- lib/git/utils.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index 2aa8d33e..cdc7c55b 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -6,6 +6,58 @@ import os +try: + import hashlib +except ImportError: + import sha + +def make_sha(source=''): + """ + A python2.4 workaround for the sha/hashlib module fiasco + + Note + From the dulwich project + """ + try: + return hashlib.sha1(source) + except NameError: + sha1 = sha.sha(source) + return sha1 + + +class SHA1Writer(object): + """ + Wrapper around a file-like object that remembers the SHA1 of + the data written to it. It will write a sha when the stream is closed + or if the asked for explicitly usign write_sha. + + Note: + Based on the dulwich project + """ + __slots__ = ("f", "sha1") + + def __init__(self, f): + self.f = f + self.sha1 = make_sha("") + + def write(self, data): + self.sha1.update(data) + self.f.write(data) + + def write_sha(self): + sha = self.sha1.digest() + self.f.write(sha) + return sha + + def close(self): + sha = self.write_sha() + self.f.close() + return sha + + def tell(self): + return self.f.tell() + + class LazyMixin(object): """ Base class providing an interface to lazily retrieve attribute values upon -- cgit v1.2.1 From 36f838e7977e62284d2928f65cacf29771f1952f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 19:05:00 +0200 Subject: utils: Added LockFile including test GitConfigFile is now derived from LockFile using its capabilities Implemented ConcurrentWriteOperation, test is yet to be done --- lib/git/utils.py | 159 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index cdc7c55b..dd1117b6 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -5,6 +5,8 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os +import sys +import tempfile try: import hashlib @@ -58,6 +60,163 @@ class SHA1Writer(object): return self.f.tell() +class LockFile(object): + """ + Provides methods to obtain, check for, and release a file based lock which + should be used to handle concurrent access to the same file. + + As we are a utility class to be derived from, we only use protected methods. + + Locks will automatically be released on destruction + """ + __slots__ = ("_file_path", "_owns_lock") + + def __init__(self, file_path): + self._file_path = file_path + self._owns_lock = False + + def __del__(self): + self._release_lock() + + def _lock_file_path(self): + """ + Return + Path to lockfile + """ + return "%s.lock" % (self._file_path) + + def _has_lock(self): + """ + Return + True if we have a lock and if the lockfile still exists + + Raise + AssertionError if our lock-file does not exist + """ + if not self._owns_lock: + return False + + lock_file = self._lock_file_path() + try: + fp = open(lock_file, "r") + pid = int(fp.read()) + fp.close() + except IOError: + raise AssertionError("GitConfigParser has a lock but the lock file at %s could not be read" % lock_file) + + if pid != os.getpid(): + raise AssertionError("We claim to own the lock at %s, but it was not owned by our process: %i" % (lock_file, os.getpid())) + + return True + + def _obtain_lock_or_raise(self): + """ + Create a lock file as flag for other instances, mark our instance as lock-holder + + Raise + IOError if a lock was already present or a lock file could not be written + """ + if self._has_lock(): + return + + lock_file = self._lock_file_path() + if os.path.exists(lock_file): + raise IOError("Lock for file %r did already exist, delete %r in case the lock is illegal" % (self._file_path, lock_file)) + + fp = open(lock_file, "w") + fp.write(str(os.getpid())) + fp.close() + + self._owns_lock = True + + def _release_lock(self): + """ + Release our lock if we have one + """ + if not self._has_lock(): + return + + os.remove(self._lock_file_path()) + self._owns_lock = False + + +class ConcurrentWriteOperation(LockFile): + """ + This class facilitates a safe write operation to a file on disk such that we: + + - lock the original file + - write to a temporary file + - rename temporary file back to the original one on close + - unlock the original file + + This type handles error correctly in that it will assure a consistent state + on destruction + """ + __slots__ = "_temp_write_fp" + + def __init__(self, file_path): + """ + Initialize an instance able to write the given file_path + """ + super(ConcurrentWriteOperation, self).__init__(file_path) + self._temp_write_fp = None + + def __del__(self): + self._end_writing(successful=False) + + def _begin_writing(self): + """ + Begin writing our file, hence we get a lock and start writing + a temporary file in the same directory. + + Returns + File Object to write to. It is still maintained by this instance + and you do not need to manually close + """ + # already writing ? + if self._temp_write_fp is not None: + return self._temp_write_fp + + self._obtain_lock_or_raise() + dirname, basename = os.path.split(self._file_path) + self._temp_write_fp = open(tempfile.mktemp(basename, '', dirname), "w") + return self._temp_write_fp + + def _is_writing(self): + """ + Returns + True if we are currently writing a file + """ + return self._temp_write_fp is not None + + def _end_writing(self, successful=True): + """ + Indicate you successfully finished writing the file to: + + - close the underlying stream + - rename the remporary file to the original one + - release our lock + """ + # did we start a write operation ? + if self._temp_write_fp is None: + return + + self._temp_write_fp.close() + if successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + os.remove(self._file_path) + os.rename(self._temp_write_fp.name, self._file_path) + else: + # just delete the file so far, we failed + os.remove(self._temp_write_fp.name) + # END successful handling + + # finally reset our handle + self._release_lock() + self._temp_write_fp = None + + class LazyMixin(object): """ Base class providing an interface to lazily retrieve attribute values upon -- cgit v1.2.1 From 7184340f9d826a52b9e72fce8a30b21a7c73d5be Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 19:23:21 +0200 Subject: Added test for ConcurrentWriteOperation --- lib/git/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index dd1117b6..f188a7a4 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -205,7 +205,10 @@ class ConcurrentWriteOperation(LockFile): if successful: # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": - os.remove(self._file_path) + if os.path.isfile(self._file_path): + os.remove(self._file_path) + # END remove if exists + # END win32 special handling os.rename(self._temp_write_fp.name, self._file_path) else: # just delete the file so far, we failed -- cgit v1.2.1 From 1496979cf7e9692ef869d2f99da6141756e08d25 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Oct 2009 19:43:01 +0200 Subject: default index writing now writes the index of the current repository in a fashion comparable to the native implementation --- lib/git/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index f188a7a4..8cdb4804 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -201,7 +201,9 @@ class ConcurrentWriteOperation(LockFile): if self._temp_write_fp is None: return - self._temp_write_fp.close() + if not self._temp_write_fp.closed: + self._temp_write_fp.close() + if successful: # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": -- cgit v1.2.1 From 146a6fe18da94e12aa46ec74582db640e3bbb3a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 28 Oct 2009 12:00:58 +0100 Subject: IterableList: added support for prefix allowing remote.refs.master constructs, previously it was remote.refs['%s/master'%remote] Added first simple test for push support, which shows that much more work is needed on that side to allow just-in-time progress information --- lib/git/utils.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'lib/git/utils.py') diff --git a/lib/git/utils.py b/lib/git/utils.py index 8cdb4804..48427ff2 100644 --- a/lib/git/utils.py +++ b/lib/git/utils.py @@ -260,16 +260,25 @@ class IterableList(list): heads.master heads['master'] heads[0] + + It requires an id_attribute name to be set which will be queried from its + contained items to have a means for comparison. + + A prefix can be specified which is to be used in case the id returned by the + items always contains a prefix that does not matter to the user, so it + can be left out. """ - __slots__ = '_id_attr' + __slots__ = ('_id_attr', '_prefix') - def __new__(cls, id_attr): + def __new__(cls, id_attr, prefix=''): return super(IterableList,cls).__new__(cls) - def __init__(self, id_attr): + def __init__(self, id_attr, prefix=''): self._id_attr = id_attr + self._prefix = prefix def __getattr__(self, attr): + attr = self._prefix + attr for item in self: if getattr(item, self._id_attr) == attr: return item @@ -283,7 +292,7 @@ class IterableList(list): try: return getattr(self, index) except AttributeError: - raise IndexError( "No item found with id %r" % index ) + raise IndexError( "No item found with id %r" % self._prefix + index ) class Iterable(object): """ -- cgit v1.2.1