From 690828ce2e03ce32c5a66186c543d7c5050287e4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 6 Jul 2011 22:11:48 +0200 Subject: Added basis for initial dulwich integration. Many basic issues should surface while integrating this --- git/db/dulwich/__init__.py | 13 +++++++++++++ git/db/dulwich/base.py | 6 ++++++ 2 files changed, 19 insertions(+) create mode 100644 git/db/dulwich/__init__.py create mode 100644 git/db/dulwich/base.py (limited to 'git/db') diff --git a/git/db/dulwich/__init__.py b/git/db/dulwich/__init__.py new file mode 100644 index 00000000..92d30941 --- /dev/null +++ b/git/db/dulwich/__init__.py @@ -0,0 +1,13 @@ +"""Dulwich module initialization""" + +def init_dulwich(): + """:raise ImportError: if dulwich is not present""" + try: + import dulwich + except ImportError: + raise ImportError("Could not find 'dulwich' in the PYTHONPATH - dulwich functionality is not available") + #END handle dulwich import + + + +init_dulwich() diff --git a/git/db/dulwich/base.py b/git/db/dulwich/base.py new file mode 100644 index 00000000..cd1d71c8 --- /dev/null +++ b/git/db/dulwich/base.py @@ -0,0 +1,6 @@ +"""Module with some basic database implementations""" + + +__all__ = [] + + -- cgit v1.2.1 From 4bb5107cff6f205f5c6e73a6f8bd22fc56f48cf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Jul 2011 14:53:37 +0200 Subject: Initial version of the DulwichType inheritance. For now, it inherits everything from the existing implementation, but one by one things can be reimplmented to use dulwich. It also shows that py 2.6 is quite plagued from its new feature, which is actually a bug, as objects inability to accept any args makes mixins hard to use ... --- git/db/compat.py | 24 +++++++++++++++---- git/db/dulwich/base.py | 6 ----- git/db/dulwich/complex.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ git/db/interface.py | 12 ++-------- git/db/py/base.py | 19 +++++++++++---- git/db/py/complex.py | 16 +++---------- git/db/py/resolve.py | 4 ---- 7 files changed, 97 insertions(+), 43 deletions(-) delete mode 100644 git/db/dulwich/base.py create mode 100644 git/db/dulwich/complex.py (limited to 'git/db') diff --git a/git/db/compat.py b/git/db/compat.py index 767ab5e0..771a1e77 100644 --- a/git/db/compat.py +++ b/git/db/compat.py @@ -4,14 +4,10 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module providing adaptors to maintain backwards compatability""" -class RepoCompatibilityInterface(object): +class RepoCompatibilityInterfaceNoBare(object): """Interface to install backwards compatability of the new complex repository types with the previous, all in one, repository.""" - @property - def bare(self): - return self.is_bare - def rev_parse(self, *args, **kwargs): return self.resolve_object(*args, **kwargs) @@ -28,4 +24,22 @@ class RepoCompatibilityInterface(object): return self.head.reference def __repr__(self): + """Return the representation of the repository, the way it used to be""" return '' % self.git_dir + + @property + def branches(self): + return self.heads + + +class RepoCompatibilityInterface(RepoCompatibilityInterfaceNoBare): + """Interface to install backwards compatability of the new complex repository + types with the previous, all in one, repository.""" + + @property + def bare(self): + return self.is_bare + + @property + def refs(self): + return self.references diff --git a/git/db/dulwich/base.py b/git/db/dulwich/base.py deleted file mode 100644 index cd1d71c8..00000000 --- a/git/db/dulwich/base.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Module with some basic database implementations""" - - -__all__ = [] - - diff --git a/git/db/dulwich/complex.py b/git/db/dulwich/complex.py new file mode 100644 index 00000000..6c3645a4 --- /dev/null +++ b/git/db/dulwich/complex.py @@ -0,0 +1,59 @@ + +__all__ = ['DulwichGitODB', 'DulwichGitDB', 'DulwichCompatibilityGitDB'] + +from git.db.py.complex import PureGitODB +from git.db.py.base import ( + PureRepositoryPathsMixin, + PureConfigurationMixin, + PureIndexDB, + ) +from git.db.py.resolve import PureReferencesMixin +from git.db.py.transport import PureTransportDB +from git.db.py.submodule import PureSubmoduleDB + +from git.db.cmd.complex import CmdHighLevelRepository, GitCommandMixin +from git.db.compat import RepoCompatibilityInterfaceNoBare + +#from git.db.interface import ObjectDBW, ObjectDBR +from dulwich.repo import Repo as DulwichRepo + +import os + + +class DulwichGitODB(PureGitODB): + """A full fledged database to read and write object files from all kinds of sources.""" + + def __init__(self, objects_root): + """Initalize this instance""" + PureGitODB.__init__(self, objects_root) + self._dw_repo = DulwichRepo(self.working_dir) + + def __getattr__(self, attr): + try: + # supply LazyMixin with this call first + return super(DulwichGitODB, self).__getattr__(attr) + except AttributeError: + # now assume its on the dulwich repository ... for now + return getattr(self._dw_repo, attr) + #END handle attr + + +class DulwichGitDB( PureRepositoryPathsMixin, PureConfigurationMixin, + PureReferencesMixin, PureSubmoduleDB, + PureIndexDB, + PureTransportDB, # not fully implemented + GitCommandMixin, + CmdHighLevelRepository, + DulwichGitODB): # must come last, as it doesn't pass on __init__ with super + + + def __init__(self, root_path): + """Initialize ourselves on the .git directory, or the .git/objects directory.""" + PureRepositoryPathsMixin._initialize(self, root_path) + super(DulwichGitDB, self).__init__(self.objects_dir) + + +class DulwichCompatibilityGitDB(RepoCompatibilityInterfaceNoBare, DulwichGitDB): + """Basic dulwich compatibility database""" + pass + diff --git a/git/db/interface.py b/git/db/interface.py index 803f7769..9ad74cc1 100644 --- a/git/db/interface.py +++ b/git/db/interface.py @@ -561,16 +561,8 @@ class ReferencesMixin(object): raise NotImplementedError() #}END edit methods - - #{ Backward Compatability - # These aliases need to be provided by the implementing interface as well - refs = references - branches = heads - #} END backward compatability - - - - + + class RepositoryPathsMixin(object): """Represents basic functionality of a full git repository. This involves an optional working tree, a git directory with references and an object directory. diff --git a/git/db/py/base.py b/git/db/py/base.py index 2c21c136..fb6e2f4a 100644 --- a/git/db/py/base.py +++ b/git/db/py/base.py @@ -104,7 +104,6 @@ class PureRootPathDB(RootPathDB): super(PureRootPathDB, self).__init__(root_path) - #{ Interface def root_path(self): return self._root_path @@ -233,7 +232,7 @@ class PureCompoundDB(CompoundDB, PureObjectDBR, LazyMixin, CachingDB): class PureRepositoryPathsMixin(RepositoryPathsMixin): # slots has no effect here, its just to keep track of used attrs - __slots__ = ("_git_path", '_bare') + __slots__ = ("_git_path", '_bare', '_working_tree_dir') #{ Configuration repo_dir = '.git' @@ -272,14 +271,16 @@ class PureRepositoryPathsMixin(RepositoryPathsMixin): raise InvalidGitRepositoryError(epath) # END path not found - self._bare = self._git_path.endswith(self.repo_dir) + self._bare = self._working_tree_dir is None if hasattr(self, 'config_reader'): try: self._bare = self.config_reader("repository").getboolean('core','bare') except Exception: # lets not assume the option exists, although it should pass + #END handle exception #END check bare flag + self._working_tree_dir = self._bare and None or self._working_tree_dir #} end subclass interface @@ -313,7 +314,7 @@ class PureRepositoryPathsMixin(RepositoryPathsMixin): @property def working_tree_dir(self): - if self.is_bare: + if self._working_tree_dir is None: raise AssertionError("Repository at %s is bare and does not have a working tree directory" % self.git_dir) #END assertion return dirname(self.git_dir) @@ -354,6 +355,10 @@ class PureConfigurationMixin(ConfigurationMixin): repo_config_file_name = "config" #} END + def __new__(cls, *args, **kwargs): + """This is just a stupid workaround for the evil py2.6 change which makes mixins quite impossible""" + return super(PureConfigurationMixin, cls).__new__(cls, *args, **kwargs) + def __init__(self, *args, **kwargs): """Verify prereqs""" try: @@ -421,7 +426,11 @@ class PureAlternatesFileMixin(object): #} END configuration def __init__(self, *args, **kwargs): - super(PureAlternatesFileMixin, self).__init__(*args, **kwargs) + try: + super(PureAlternatesFileMixin, self).__init__(*args, **kwargs) + except TypeError: + pass + #END handle py2.6 code breaking changes self._alternates_path() # throws on incompatible type #{ Interface diff --git a/git/db/py/complex.py b/git/db/py/complex.py index d5c185f3..5f4e81e0 100644 --- a/git/db/py/complex.py +++ b/git/db/py/complex.py @@ -22,17 +22,7 @@ from submodule import PureSubmoduleDB from git.db.compat import RepoCompatibilityInterface -from git.util import ( - LazyMixin, - normpath, - join, - dirname - ) -from git.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) +from git.exc import InvalidDBRoot import os __all__ = ('PureGitODB', 'PurePartialGitDB', 'PureCompatibilityGitDB') @@ -106,7 +96,8 @@ class PureGitODB(PureRootPathDB, PureObjectDBW, PureCompoundDB, PureAlternatesFi class PurePartialGitDB(PureGitODB, PureRepositoryPathsMixin, PureConfigurationMixin, PureReferencesMixin, PureSubmoduleDB, - PureIndexDB, PureTransportDB + PureIndexDB, + PureTransportDB # not fully implemented # HighLevelRepository Currently not implemented ! ): """Git like database with support for object lookup as well as reference resolution. @@ -122,7 +113,6 @@ class PurePartialGitDB(PureGitODB, super(PurePartialGitDB, self).__init__(self.objects_dir) - class PureCompatibilityGitDB(PurePartialGitDB, RepoCompatibilityInterface): """Pure git database with a compatability layer required by 0.3x code""" diff --git a/git/db/py/resolve.py b/git/db/py/resolve.py index 7bea779e..9a31fbd8 100644 --- a/git/db/py/resolve.py +++ b/git/db/py/resolve.py @@ -361,7 +361,3 @@ class PureReferencesMixin(ReferencesMixin): def delete_tag(self, *tags): return self.TagReferenceCls.delete(self, *tags) - - # compat - branches = heads - refs = references -- cgit v1.2.1 From 8b3b222565199eab67196a5ab840b9e2770bfc53 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Jul 2011 19:34:25 +0200 Subject: Added default performance tests - these should help to measure something at least, which implicitly includes pack handling. For the pack specific tests to work, one would need a pack interface though, which is currently not planned to be specifically exposed --- git/db/dulwich/complex.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'git/db') diff --git a/git/db/dulwich/complex.py b/git/db/dulwich/complex.py index 6c3645a4..3fa7c1cd 100644 --- a/git/db/dulwich/complex.py +++ b/git/db/dulwich/complex.py @@ -26,7 +26,12 @@ class DulwichGitODB(PureGitODB): def __init__(self, objects_root): """Initalize this instance""" PureGitODB.__init__(self, objects_root) - self._dw_repo = DulwichRepo(self.working_dir) + if hasattr(self, 'working_dir'): + wd = self.working_dir + else: + wd = os.path.dirname(os.path.dirname(objects_root)) + #END try to figure out good entry for dulwich, which doesn't do an extensive search + self._dw_repo = DulwichRepo(wd) def __getattr__(self, attr): try: -- cgit v1.2.1 From 09064504e52a5ec8bfc4825a3176239b731380d2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Jul 2011 21:28:08 +0200 Subject: Added trivial implementation for info and stream methods - info is very inefficient, but can't help it. Basic repo tests don't work as dulwich ignores alternate files --- git/db/dulwich/complex.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'git/db') diff --git a/git/db/dulwich/complex.py b/git/db/dulwich/complex.py index 3fa7c1cd..e1dad01d 100644 --- a/git/db/dulwich/complex.py +++ b/git/db/dulwich/complex.py @@ -17,6 +17,10 @@ from git.db.compat import RepoCompatibilityInterfaceNoBare #from git.db.interface import ObjectDBW, ObjectDBR from dulwich.repo import Repo as DulwichRepo +from git.base import OInfo, OStream +from git.fun import type_id_to_type_map + +from cStringIO import StringIO import os @@ -42,6 +46,18 @@ class DulwichGitODB(PureGitODB): return getattr(self._dw_repo, attr) #END handle attr + #{ Object DBR + + def info(self, binsha): + type_id, uncomp_data = self._dw_repo.object_store.get_raw(binsha) + return OInfo(binsha, type_id_to_type_map[type_id], len(uncomp_data)) + + def stream(self, binsha): + type_id, uncomp_data = self._dw_repo.object_store.get_raw(binsha) + return OStream(binsha, type_id_to_type_map[type_id], len(uncomp_data), StringIO(uncomp_data)) + + #}END object dbr + class DulwichGitDB( PureRepositoryPathsMixin, PureConfigurationMixin, PureReferencesMixin, PureSubmoduleDB, -- cgit v1.2.1 From 6a10c8a82adcbe1f18c2e79a9135c7f5c753b826 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Jul 2011 21:47:02 +0200 Subject: Removed cache in PureCompoundDB as it had the tendency to slow things down actually --- git/db/py/base.py | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) (limited to 'git/db') diff --git a/git/db/py/base.py b/git/db/py/base.py index fb6e2f4a..49f28a8d 100644 --- a/git/db/py/base.py +++ b/git/db/py/base.py @@ -131,44 +131,33 @@ class PureCompoundDB(CompoundDB, PureObjectDBR, LazyMixin, CachingDB): def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() - elif attr == '_obj_cache': - self._obj_cache = dict() else: super(PureCompoundDB, self)._set_cache_(attr) - def _db_query(self, sha): - """:return: database containing the given 20 byte sha - :raise BadObject:""" - # most databases use binary representations, prevent converting - # it everytime a database is being queried - try: - return self._obj_cache[sha] - except KeyError: - pass - # END first level cache - - for db in self._dbs: - if db.has_object(sha): - self._obj_cache[sha] = db - return db - # END for each database - raise BadObject(sha) - #{ PureObjectDBR interface def has_object(self, sha): - try: - self._db_query(sha) - return True - except BadObject: - return False - # END handle exceptions + for db in self._dbs: + if db.has_object(sha): + return True + #END for each db + return False def info(self, sha): - return self._db_query(sha).info(sha) + for db in self._dbs: + try: + return db.info(sha) + except BadObject: + pass + #END for each db def stream(self, sha): - return self._db_query(sha).stream(sha) + for db in self._dbs: + try: + return db.stream(sha) + except BadObject: + pass + #END for each db def size(self): return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) @@ -185,7 +174,6 @@ class PureCompoundDB(CompoundDB, PureObjectDBR, LazyMixin, CachingDB): def update_cache(self, force=False): # something might have changed, clear everything - self._obj_cache.clear() stat = False for db in self._dbs: if isinstance(db, CachingDB): -- cgit v1.2.1 From f4f330f8588dacd43af6513e1e1e1a50237da1e7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Jul 2011 23:37:04 +0200 Subject: Added store support. Now the basic object IO is implemented, which shall be enough for the first batch of work --- git/db/dulwich/complex.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'git/db') diff --git a/git/db/dulwich/complex.py b/git/db/dulwich/complex.py index e1dad01d..ad5b97a4 100644 --- a/git/db/dulwich/complex.py +++ b/git/db/dulwich/complex.py @@ -16,9 +16,10 @@ from git.db.compat import RepoCompatibilityInterfaceNoBare #from git.db.interface import ObjectDBW, ObjectDBR from dulwich.repo import Repo as DulwichRepo +from dulwich.objects import ShaFile from git.base import OInfo, OStream -from git.fun import type_id_to_type_map +from git.fun import type_id_to_type_map, type_to_type_id_map from cStringIO import StringIO import os @@ -57,7 +58,16 @@ class DulwichGitODB(PureGitODB): return OStream(binsha, type_id_to_type_map[type_id], len(uncomp_data), StringIO(uncomp_data)) #}END object dbr + + #{ Object DBW + + def store(self, istream): + obj = ShaFile.from_raw_string(type_to_type_id_map[istream.type], istream.read()) + self._dw_repo.object_store.add_object(obj) + istream.binsha = obj.sha().digest() + return istream + #}END object dbw class DulwichGitDB( PureRepositoryPathsMixin, PureConfigurationMixin, PureReferencesMixin, PureSubmoduleDB, -- cgit v1.2.1