diff options
| -rw-r--r-- | doc/source/changes.rst | 8 | ||||
| -rw-r--r-- | objects/submodule/base.py | 166 | ||||
| -rw-r--r-- | objects/submodule/root.py | 286 | ||||
| -rw-r--r-- | remote.py | 121 | ||||
| -rw-r--r-- | test/test_submodule.py | 69 | ||||
| -rw-r--r-- | util.py | 123 |
6 files changed, 478 insertions, 295 deletions
diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d820c8ca..74692955 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -21,15 +21,11 @@ Changelog * ``create(...)`` method now supports the reflog, but will not raise ``GitCommandError`` anymore as it is a pure python implementation now. Instead, it raises ``OSError``. - * **Intrusive Changes** to ``Actor`` type - - * the *name* field is now using unicode if ascii does not match - * **Intrusive Changes** to ``Repo`` type - * ``create_head(...)`` method does not support **kwargs anymore, instead it supports a logmsg parameter + * ``create_head(...)`` method does not support kwargs anymore, instead it supports a logmsg parameter -* Repo.rev_parse now supports the [ref]@{n} syntax, where n is the number of steps to look into the reference's past +* Repo.rev_parse now supports the [ref]@{n} syntax, where *n* is the number of steps to look into the reference's past * **BugFixes** diff --git a/objects/submodule/base.py b/objects/submodule/base.py index 36b48d78..88cc42da 100644 --- a/objects/submodule/base.py +++ b/objects/submodule/base.py @@ -12,7 +12,8 @@ from StringIO import StringIO # need a dict to set bloody .name field from git.util import ( Iterable, join_path_native, - to_native_path_linux + to_native_path_linux, + RemoteProgress ) from git.config import SectionConstraint @@ -20,6 +21,7 @@ from git.exc import ( InvalidGitRepositoryError, NoSuchPathError ) + import stat import git @@ -29,7 +31,23 @@ import time import shutil -__all__ = ["Submodule"] +__all__ = ["Submodule", "UpdateProgress"] + + +class UpdateProgress(RemoteProgress): + """Class providing detailed progress information to the caller who should + derive from it and implement the ``update(...)`` message""" + CLONE, FETCH, UPDWKTREE = [1 << x for x in range(RemoteProgress._num_op_codes, RemoteProgress._num_op_codes+3)] + _num_op_codes = RemoteProgress._num_op_codes + 3 + + __slots__ = tuple() + + +BEGIN = UpdateProgress.BEGIN +END = UpdateProgress.END +CLONE = UpdateProgress.CLONE +FETCH = UpdateProgress.FETCH +UPDWKTREE = UpdateProgress.UPDWKTREE # IndexObject comes via util module, its a 'hacky' fix thanks to pythons import @@ -285,7 +303,8 @@ class Submodule(util.IndexObject, Iterable, Traversable): return sm - def update(self, recursive=False, init=True, to_latest_revision=False): + def update(self, recursive=False, init=True, to_latest_revision=False, progress=None, + dry_run=False): """Update the repository of this submodule to point to the checkout we point at with the binsha of this instance. @@ -297,6 +316,9 @@ class Submodule(util.IndexObject, Iterable, Traversable): This only works if we have a local tracking branch, which is the case if the remote repository had a master branch, or of the 'branch' option was specified for this submodule and the branch existed remotely + :param progress: UpdateProgress instance or None of no progress should be shown + :param dry_run: if True, the operation will only be simulated, but not performed. + All performed operations are read-only :note: does nothing in bare repositories :note: method is definitely not atomic if recurisve is True :return: self""" @@ -304,13 +326,41 @@ class Submodule(util.IndexObject, Iterable, Traversable): return self #END pass in bare mode + if progress is None: + progress = UpdateProgress() + #END handle progress + prefix = '' + if dry_run: + prefix = "DRY-RUN: " + #END handle prefix + + # to keep things plausible in dry-run mode + if dry_run: + mrepo = None + #END init mrepo # ASSURE REPO IS PRESENT AND UPTODATE ##################################### try: mrepo = self.module() - for remote in mrepo.remotes: - remote.fetch() + rmts = mrepo.remotes + len_rmts = len(rmts) + for i, remote in enumerate(rmts): + op = FETCH + if i == 0: + op |= BEGIN + #END handle start + + progress.update(op, i, len_rmts, prefix+"Fetching remote %s of submodule %r" % (remote, self.name)) + #=============================== + if not dry_run: + remote.fetch(progress=progress) + #END handle dry-run + #=============================== + if i == len_rmts-1: + op |= END + #END handle end + progress.update(op, i, len_rmts, prefix+"Done fetching remote of submodule %r" % self.name) #END fetch new data except InvalidGitRepositoryError: if not init: @@ -320,7 +370,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): # there is no git-repository yet - but delete empty paths module_path = join_path_native(self.repo.working_tree_dir, self.path) - if os.path.isdir(module_path): + if not dry_run and os.path.isdir(module_path): try: os.rmdir(module_path) except OSError: @@ -330,31 +380,38 @@ class Submodule(util.IndexObject, Iterable, Traversable): # don't check it out at first - nonetheless it will create a local # branch according to the remote-HEAD if possible - mrepo = git.Repo.clone_from(self.url, module_path, n=True) + progress.update(BEGIN|CLONE, 0, 1, prefix+"Cloning %s to %s in submodule %r" % (self.url, module_path, self.name)) + if not dry_run: + mrepo = git.Repo.clone_from(self.url, module_path, n=True) + #END handle dry-run + progress.update(END|CLONE, 0, 1, prefix+"Done cloning to %s" % module_path) - # see whether we have a valid branch to checkout - try: - # find a remote which has our branch - we try to be flexible - remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) - local_branch = mkhead(mrepo, self.branch_path) - - # have a valid branch, but no checkout - make sure we can figure - # that out by marking the commit with a null_sha - local_branch.set_object(util.Object(mrepo, self.NULL_BIN_SHA)) - # END initial checkout + branch creation - - # make sure HEAD is not detached - mrepo.head.set_reference(local_branch, logmsg="submodule: attaching head to %s" % local_branch) - mrepo.head.ref.set_tracking_branch(remote_branch) - except IndexError: - print >> sys.stderr, "Warning: Failed to checkout tracking branch %s" % self.branch_path - #END handle tracking branch - # NOTE: Have to write the repo config file as well, otherwise - # the default implementation will be offended and not update the repository - # Maybe this is a good way to assure it doesn't get into our way, but - # we want to stay backwards compatible too ... . Its so redundant ! - self.repo.config_writer().set_value(sm_section(self.name), 'url', self.url) + if not dry_run: + # see whether we have a valid branch to checkout + try: + # find a remote which has our branch - we try to be flexible + remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) + local_branch = mkhead(mrepo, self.branch_path) + + # have a valid branch, but no checkout - make sure we can figure + # that out by marking the commit with a null_sha + local_branch.set_object(util.Object(mrepo, self.NULL_BIN_SHA)) + # END initial checkout + branch creation + + # make sure HEAD is not detached + mrepo.head.set_reference(local_branch, logmsg="submodule: attaching head to %s" % local_branch) + mrepo.head.ref.set_tracking_branch(remote_branch) + except IndexError: + print >> sys.stderr, "Warning: Failed to checkout tracking branch %s" % self.branch_path + #END handle tracking branch + + # NOTE: Have to write the repo config file as well, otherwise + # the default implementation will be offended and not update the repository + # Maybe this is a good way to assure it doesn't get into our way, but + # we want to stay backwards compatible too ... . Its so redundant ! + self.repo.config_writer().set_value(sm_section(self.name), 'url', self.url) + #END handle dry_run #END handle initalization @@ -362,8 +419,12 @@ class Submodule(util.IndexObject, Iterable, Traversable): ############################ binsha = self.binsha hexsha = self.hexsha - is_detached = mrepo.head.is_detached - if to_latest_revision: + if mrepo is not None: + # mrepo is only set if we are not in dry-run mode or if the module existed + is_detached = mrepo.head.is_detached + #END handle dry_run + + if not dry_run and to_latest_revision: msg_base = "Cannot update to latest revision in repository at %r as " % mrepo.working_dir if not is_detached: rref = mrepo.head.ref.tracking_branch() @@ -380,27 +441,35 @@ class Submodule(util.IndexObject, Iterable, Traversable): # END handle to_latest_revision option # update the working tree - if mrepo.head.commit.binsha != binsha: - if is_detached: - # NOTE: for now we force, the user is no supposed to change detached - # submodules anyway. Maybe at some point this becomes an option, to - # properly handle user modifications - see below for future options - # regarding rebase and merge. - mrepo.git.checkout(hexsha, force=True) - else: - # TODO: allow to specify a rebase, merge, or reset - # TODO: Warn if the hexsha forces the tracking branch off the remote - # branch - this should be prevented when setting the branch option - mrepo.head.reset(hexsha, index=True, working_tree=True) - # END handle checkout + # handles dry_run + if mrepo is not None and mrepo.head.commit.binsha != binsha: + progress.update(BEGIN|UPDWKTREE, 0, 1, prefix+"Updating working tree at %s for submodule %r" % (self.path, self.name)) + if not dry_run: + if is_detached: + # NOTE: for now we force, the user is no supposed to change detached + # submodules anyway. Maybe at some point this becomes an option, to + # properly handle user modifications - see below for future options + # regarding rebase and merge. + mrepo.git.checkout(hexsha, force=True) + else: + # TODO: allow to specify a rebase, merge, or reset + # TODO: Warn if the hexsha forces the tracking branch off the remote + # branch - this should be prevented when setting the branch option + mrepo.head.reset(hexsha, index=True, working_tree=True) + # END handle checkout + #END handle dry_run + progress.update(END|UPDWKTREE, 0, 1, prefix+"Done updating working tree for submodule %r" % self.name) # END update to new commit only if needed # HANDLE RECURSION ################## if recursive: - for submodule in self.iter_items(self.module()): - submodule.update(recursive, init, to_latest_revision) - # END handle recursive update + # in dry_run mode, the module might not exist + if mrepo is not None: + for submodule in self.iter_items(self.module()): + submodule.update(recursive, init, to_latest_revision, progress=progress, dry_run=dry_run) + # END handle recursive update + #END handle dry run # END for each submodule return self @@ -800,8 +869,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): def children(self): """ :return: IterableList(Submodule, ...) an iterable list of submodules instances - which are children of this submodule - :raise InvalidGitRepositoryError: if the submodule is not checked-out""" + which are children of this submodule or 0 if the submodule is not checked out""" return self._get_intermediate_items(self) #} END query interface diff --git a/objects/submodule/root.py b/objects/submodule/root.py index 753c6df4..36cd7209 100644 --- a/objects/submodule/root.py +++ b/objects/submodule/root.py @@ -1,4 +1,4 @@ -from base import Submodule +from base import Submodule, UpdateProgress from util import ( find_first_remote_branch ) @@ -7,9 +7,23 @@ import git import sys -__all__ = ["RootModule"] +__all__ = ["RootModule", "RootUpdateProgress"] + +class RootUpdateProgress(UpdateProgress): + """Utility class which adds more opcodes to the UpdateProgress""" + REMOVE, PATHCHANGE, BRANCHCHANGE, URLCHANGE = [1 << x for x in range(UpdateProgress._num_op_codes, UpdateProgress._num_op_codes+4)] + _num_op_codes = UpdateProgress._num_op_codes+4 + __slots__ = tuple() + +BEGIN = RootUpdateProgress.BEGIN +END = RootUpdateProgress.END +REMOVE = RootUpdateProgress.REMOVE +BRANCHCHANGE = RootUpdateProgress.BRANCHCHANGE +URLCHANGE = RootUpdateProgress.URLCHANGE +PATHCHANGE = RootUpdateProgress.PATHCHANGE + class RootModule(Submodule): """A (virtual) Root of all submodules in the given repository. It can be used to more easily traverse all submodules of the master repository""" @@ -38,7 +52,8 @@ class RootModule(Submodule): #{ Interface - def update(self, previous_commit=None, recursive=True, force_remove=False, init=True, to_latest_revision=False): + def update(self, previous_commit=None, recursive=True, force_remove=False, init=True, + to_latest_revision=False, progress=None, dry_run=False): """Update the submodules of this repository to the current HEAD commit. This method behaves smartly by determining changes of the path of a submodules repository, next to changes to the to-be-checked-out commit or the branch to be @@ -57,15 +72,26 @@ class RootModule(Submodule): :param init: If we encounter a new module which would need to be initialized, then do it. :param to_latest_revision: If True, instead of checking out the revision pointed to by this submodule's sha, the checked out tracking branch will be merged with the - newest remote branch fetched from the repository's origin""" + newest remote branch fetched from the repository's origin + :param progress: RootUpdateProgress instance or None if no progress should be sent + :param dry_run: if True, operations will not actually be performed. Progress messages + will change accordingly to indicate the WOULD DO state of the operation.""" if self.repo.bare: raise InvalidGitRepositoryError("Cannot update submodules in bare repositories") # END handle bare + if progress is None: + progress = RootUpdateProgress() + #END assure progress is set + + prefix = '' + if dry_run: + prefix = 'DRY-RUN: ' + repo = self.repo - # HANDLE COMMITS - ################## + # SETUP BASE COMMIT + ################### cur_commit = repo.head.commit if previous_commit is None: try: @@ -89,28 +115,51 @@ class RootModule(Submodule): # HANDLE REMOVALS ################### - for rsm in (spsms - ssms): + rrsm = (spsms - ssms) + len_rrsm = len(rrsm) + for i, rsm in enumerate(rrsm): + op = REMOVE + if i == 0: + op |= BEGIN + #END handle begin + # fake it into thinking its at the current commit to allow deletion # of previous module. Trigger the cache to be updated before that - #rsm.url + progress.update(op, i, len_rrsm, prefix+"Removing submodule %r at %s" % (rsm.name, rsm.abspath)) rsm._parent_commit = repo.head.commit - rsm.remove(configuration=False, module=True, force=force_remove) + if not dry_run: + rsm.remove(configuration=False, module=True, force=force_remove) + #END handle dry-run + + if i == len_rrsm-1: + op |= END + #END handle end + progress.update(op, i, len_rrsm, prefix+"Done removing submodule %r" % rsm.name) # END for each removed submodule # HANDLE PATH RENAMES ##################### # url changes + branch changes - for csm in (spsms & ssms): + csms = (spsms & ssms) + len_csms = len(csms) + for i, csm in enumerate(csms): psm = psms[csm.name] sm = sms[csm.name] + #PATH CHANGES + ############## if sm.path != psm.path and psm.module_exists(): + progress.update(BEGIN|PATHCHANGE, i, len_csms, prefix+"Moving repository of submodule %r from %s to %s" % (sm.name, psm.abspath, sm.abspath)) # move the module to the new path - psm.move(sm.path, module=True, configuration=False) + if not dry_run: + psm.move(sm.path, module=True, configuration=False) + #END handle dry_run + progress.update(END|PATHCHANGE, i, len_csms, prefix+"Done moving repository of submodule %r" % sm.name) # END handle path changes if sm.module_exists(): - # handle url change + # HANDLE URL CHANGE + ################### if sm.url != psm.url: # Add the new remote, remove the old one # This way, if the url just changes, the commits will not @@ -121,111 +170,119 @@ class RootModule(Submodule): # don't do anything if we already have the url we search in place if len([r for r in rmts if r.url == sm.url]) == 0: + progress.update(BEGIN|URLCHANGE, i, len_csms, prefix+"Changing url of submodule %r from %s to %s" % (sm.name, psm.url, sm.url)) - - assert nn not in [r.name for r in rmts] - smr = smm.create_remote(nn, sm.url) - smr.fetch() - - # If we have a tracking branch, it should be available - # in the new remote as well. - if len([r for r in smr.refs if r.remote_head == sm.branch_name]) == 0: - raise ValueError("Submodule branch named %r was not available in new submodule remote at %r" % (sm.branch_name, sm.url)) - # END head is not detached - - # now delete the changed one - rmt_for_deletion = None - for remote in rmts: - if remote.url == psm.url: - rmt_for_deletion = remote - break - # END if urls match - # END for each remote - - # if we didn't find a matching remote, but have exactly one, - # we can safely use this one - if rmt_for_deletion is None: - if len(rmts) == 1: - rmt_for_deletion = rmts[0] - else: - # if we have not found any remote with the original url - # we may not have a name. This is a special case, - # and its okay to fail here - # Alternatively we could just generate a unique name and leave all - # existing ones in place - raise InvalidGitRepositoryError("Couldn't find original remote-repo at url %r" % psm.url) - #END handle one single remote - # END handle check we found a remote - - orig_name = rmt_for_deletion.name - smm.delete_remote(rmt_for_deletion) - # NOTE: Currently we leave tags from the deleted remotes - # as well as separate tracking branches in the possibly totally - # changed repository ( someone could have changed the url to - # another project ). At some point, one might want to clean - # it up, but the danger is high to remove stuff the user - # has added explicitly - - # rename the new remote back to what it was - smr.rename(orig_name) - - # early on, we verified that the our current tracking branch - # exists in the remote. Now we have to assure that the - # sha we point to is still contained in the new remote - # tracking branch. - smsha = sm.binsha - found = False - rref = smr.refs[self.branch_name] - for c in rref.commit.traverse(): - if c.binsha == smsha: - found = True - break - # END traverse all commits in search for sha - # END for each commit - - if not found: - # adjust our internal binsha to use the one of the remote - # this way, it will be checked out in the next step - # This will change the submodule relative to us, so - # the user will be able to commit the change easily - print >> sys.stderr, "WARNING: Current sha %s was not contained in the tracking branch at the new remote, setting it the the remote's tracking branch" % sm.hexsha - sm.binsha = rref.commit.binsha - #END reset binsha - - #NOTE: All checkout is performed by the base implementation of update - + if not dry_run: + assert nn not in [r.name for r in rmts] + smr = smm.create_remote(nn, sm.url) + smr.fetch(progress=progress) + + # If we have a tracking branch, it should be available + # in the new remote as well. + if len([r for r in smr.refs if r.remote_head == sm.branch_name]) == 0: + raise ValueError("Submodule branch named %r was not available in new submodule remote at %r" % (sm.branch_name, sm.url)) + # END head is not detached + + # now delete the changed one + rmt_for_deletion = None + for remote in rmts: + if remote.url == psm.url: + rmt_for_deletion = remote + break + # END if urls match + # END for each remote + + # if we didn't find a matching remote, but have exactly one, + # we can safely use this one + if rmt_for_deletion is None: + if len(rmts) == 1: + rmt_for_deletion = rmts[0] + else: + # if we have not found any remote with the original url + # we may not have a name. This is a special case, + # and its okay to fail here + # Alternatively we could just generate a unique name and leave all + # existing ones in place + raise InvalidGitRepositoryError("Couldn't find original remote-repo at url %r" % psm.url) + #END handle one single remote + # END handle check we found a remote + + orig_name = rmt_for_deletion.name + smm.delete_remote(rmt_for_deletion) + # NOTE: Currently we leave tags from the deleted remotes + # as well as separate tracking branches in the possibly totally + # changed repository ( someone could have changed the url to + # another project ). At some point, one might want to clean + # it up, but the danger is high to remove stuff the user + # has added explicitly + + # rename the new remote back to what it was + smr.rename(orig_name) + + # early on, we verified that the our current tracking branch + # exists in the remote. Now we have to assure that the + # sha we point to is still contained in the new remote + # tracking branch. + smsha = sm.binsha + found = False + rref = smr.refs[self.branch_name] + for c in rref.commit.traverse(): + if c.binsha == smsha: + found = True + break + # END traverse all commits in search for sha + # END for each commit + + if not found: + # adjust our internal binsha to use the one of the remote + # this way, it will be checked out in the next step + # This will change the submodule relative to us, so + # the user will be able to commit the change easily + print >> sys.stderr, "WARNING: Current sha %s was not contained in the tracking branch at the new remote, setting it the the remote's tracking branch" % sm.hexsha + sm.binsha = rref.commit.binsha + #END reset binsha + + #NOTE: All checkout is performed by the base implementation of update + #END handle dry_run + progress.update(END|URLCHANGE, i, len_csms, prefix+"Done adjusting url of submodule %r" % (sm.name)) # END skip remote handling if new url already exists in module # END handle url + # HANDLE PATH CHANGES + ##################### if sm.branch_path != psm.branch_path: # finally, create a new tracking branch which tracks the # new remote branch - smm = sm.module() - smmr = smm.remotes - try: - tbr = git.Head.create(smm, sm.branch_name, logmsg='branch: Created from HEAD') - except OSError: - # ... or reuse the existing one - tbr = git.Head(smm, sm.branch_path) - #END assure tracking branch exists - - tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name)) - # figure out whether the previous tracking branch contains - # new commits compared to the other one, if not we can - # delete it. - try: - tbr = find_first_remote_branch(smmr, psm.branch_name) - if len(smm.git.cherry(tbr, psm.branch)) == 0: - psm.branch.delete(smm, psm.branch) - #END delete original tracking branch if there are no changes - except InvalidGitRepositoryError: - # ignore it if the previous branch couldn't be found in the - # current remotes, this just means we can't handle it - pass - # END exception handling - - #NOTE: All checkout is done in the base implementation of update + progress.update(BEGIN|BRANCHCHANGE, i, len_csms, prefix+"Changing branch of submodule %r from %s to %s" % (sm.name, psm.branch_path, sm.branch_path)) + if not dry_run: + smm = sm.module() + smmr = smm.remotes + try: + tbr = git.Head.create(smm, sm.branch_name, logmsg='branch: Created from HEAD') + except OSError: + # ... or reuse the existing one + tbr = git.Head(smm, sm.branch_path) + #END assure tracking branch exists + + tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name)) + # figure out whether the previous tracking branch contains + # new commits compared to the other one, if not we can + # delete it. + try: + tbr = find_first_remote_branch(smmr, psm.branch_name) + if len(smm.git.cherry(tbr, psm.branch)) == 0: + psm.branch.delete(smm, psm.branch) + #END delete original tracking branch if there are no changes + except InvalidGitRepositoryError: + # ignore it if the previous branch couldn't be found in the + # current remotes, this just means we can't handle it + pass + # END exception handling + + #NOTE: All checkout is done in the base implementation of update + #END handle dry_run + progress.update(END|BRANCHCHANGE, i, len_csms, prefix+"Done changing branch of submodule %r" % sm.name) #END handle branch #END handle # END for each common submodule @@ -234,15 +291,20 @@ class RootModule(Submodule): ###################################### for sm in sms: # update the submodule using the default method - sm.update(recursive=False, init=init, to_latest_revision=to_latest_revision) + sm.update(recursive=False, init=init, to_latest_revision=to_latest_revision, + progress=progress, dry_run=dry_run) # update recursively depth first - question is which inconsitent # state will be better in case it fails somewhere. Defective branch # or defective depth. The RootSubmodule type will never process itself, # which was done in the previous expression if recursive: - type(self)(sm.module()).update(recursive=True, force_remove=force_remove, - init=init, to_latest_revision=to_latest_revision) + # the module would exist by now if we are not in dry_run mode + if sm.module_exists(): + type(self)(sm.module()).update( recursive=True, force_remove=force_remove, + init=init, to_latest_revision=to_latest_revision, + progress=progress, dry_run=dry_run) + #END handle dry_run #END handle recursive # END for each submodule to update @@ -7,14 +7,14 @@ # Module implementing a remote object allowing easy access to git remotes from exc import GitCommandError -from objects import Commit from ConfigParser import NoOptionError from config import SectionConstraint from git.util import ( LazyMixin, Iterable, - IterableList + IterableList, + RemoteProgress ) from refs import ( @@ -33,123 +33,6 @@ import sys __all__ = ('RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote') -class RemoteProgress(object): - """ - Handler providing an interface to parse progress information emitted by git-push - and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. - """ - BEGIN, END, COUNTING, COMPRESSING, WRITING = [ 1 << x for x in range(5) ] - STAGE_MASK = BEGIN|END - OP_MASK = COUNTING|COMPRESSING|WRITING - - __slots__ = ("_cur_line", "_seen_ops") - re_op_absolute = re.compile("(remote: )?([\w\s]+):\s+()(\d+)()(.*)") - re_op_relative = re.compile("(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)") - - def __init__(self): - self._seen_ops = list() - - def _parse_progress_line(self, line): - """Parse progress information from the given line as retrieved by git-push - or git-fetch - - :return: list(line, ...) list of lines that could not be processed""" - # handle - # Counting objects: 4, done. - # Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done. - self._cur_line = line - sub_lines = line.split('\r') - failed_lines = list() - for sline in sub_lines: - # find esacpe characters and cut them away - regex will not work with - # them as they are non-ascii. As git might expect a tty, it will send them - last_valid_index = None - for i,c in enumerate(reversed(sline)): - if ord(c) < 32: - # its a slice index - last_valid_index = -i-1 - # END character was non-ascii - # END for each character in sline - if last_valid_index is not None: - sline = sline[:last_valid_index] - # END cut away invalid part - sline = sline.rstrip() - - cur_count, max_count = None, None - match = self.re_op_relative.match(sline) - if match is None: - match = self.re_op_absolute.match(sline) - - if not match: - self.line_dropped(sline) - failed_lines.append(sline) - continue - # END could not get match - - op_code = 0 - remote, op_name, percent, cur_count, max_count, message = match.groups() - - # get operation id - if op_name == "Counting objects": - op_code |= self.COUNTING - elif op_name == "Compressing objects": - op_code |= self.COMPRESSING - elif op_name == "Writing objects": - op_code |= self.WRITING - else: - raise ValueError("Operation name %r unknown" % op_name) - - # figure out stage - if op_code not in self._seen_ops: - self._seen_ops.append(op_code) - op_code |= self.BEGIN - # END begin opcode - - if message is None: - message = '' - # END message handling - - message = message.strip() - done_token = ', done.' - if message.endswith(done_token): - op_code |= self.END - message = message[:-len(done_token)] - # END end message handling - - self.update(op_code, cur_count, max_count, message) - # END for each sub line - return failed_lines - - def line_dropped(self, line): - """Called whenever a line could not be understood and was therefore dropped.""" - pass - - def update(self, op_code, cur_count, max_count=None, message=''): - """Called whenever the progress changes - - :param op_code: - Integer allowing to be compared against Operation IDs and stage IDs. - - Stage IDs are BEGIN and END. BEGIN will only be set once for each Operation - ID as well as END. It may be that BEGIN and END are set at once in case only - one progress message was emitted due to the speed of the operation. - Between BEGIN and END, none of these flags will be set - - Operation IDs are all held within the OP_MASK. Only one Operation ID will - be active per call. - :param cur_count: Current absolute count of items - - :param max_count: - The maximum count of items we expect. It may be None in case there is - no maximum number of items or if it is (yet) unknown. - - :param message: - In case of the 'WRITING' operation, it contains the amount of bytes - transferred. It may possibly be used for other purposes as well. - - You may read the contents of the current line in self._cur_line""" - pass - class PushInfo(object): """ diff --git a/test/test_submodule.py b/test/test_submodule.py index f69c27ea..b8a25e02 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -4,12 +4,20 @@ from git.test.lib import * from git.exc import * from git.objects.submodule.base import Submodule -from git.objects.submodule.root import RootModule +from git.objects.submodule.root import RootModule, RootUpdateProgress from git.util import to_native_path_linux, join_path_native import shutil import git import os +class TestRootProgress(RootUpdateProgress): + """Just prints messages, for now without checking the correctness of the states""" + + def update(self, op, index, max_count, message=''): + print message + +prog = TestRootProgress() + class TestSubmodule(TestBase): k_subm_current = "83a9e4a0dad595188ff3fb35bc3dfc4d931eff6d" @@ -130,6 +138,10 @@ class TestSubmodule(TestBase): self.failUnlessRaises(OSError, sm.update) os.rmdir(newdir) + # dry-run does nothing + sm.update(dry_run=True, progress=prog) + assert not sm.module_exists() + assert sm.update() is sm sm_repopath = sm.path # cache for later assert sm.module_exists() @@ -150,6 +162,11 @@ class TestSubmodule(TestBase): # delete the whole directory and re-initialize shutil.rmtree(sm.abspath) + assert len(sm.children()) == 0 + # dry-run does nothing + sm.update(dry_run=True, recursive=False, progress=prog) + assert len(sm.children()) == 0 + sm.update(recursive=False) assert len(list(rwrepo.iter_submodules())) == 2 assert len(sm.children()) == 1 # its not checked out yet @@ -162,8 +179,14 @@ class TestSubmodule(TestBase): csm.config_writer().set_value('url', new_csmclone_path) assert csm.url == new_csmclone_path + # dry-run does nothing + assert not csm.module_exists() + sm.update(recursive=True, dry_run=True, progress=prog) + assert not csm.module_exists() + # update recuesively again sm.update(recursive=True) + assert csm.module_exists() # tracking branch once again csm.module().head.ref.tracking_branch() is not None @@ -175,9 +198,15 @@ class TestSubmodule(TestBase): # reset both heads to the previous version, verify that to_latest_revision works smods = (sm.module(), csm.module()) for repo in smods: - repo.head.reset('HEAD~1', working_tree=1) + repo.head.reset('HEAD~2', working_tree=1) # END for each repo to reset + # dry run does nothing + sm.update(recursive=True, dry_run=True, progress=prog) + for repo in smods: + assert repo.head.commit != repo.head.ref.tracking_branch().commit + # END for each repo to check + sm.update(recursive=True, to_latest_revision=True) for repo in smods: assert repo.head.commit == repo.head.ref.tracking_branch().commit @@ -378,6 +407,11 @@ class TestSubmodule(TestBase): # assure we clone from a local source sm.config_writer().set_value('url', to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path))) + + # dry-run does nothing + sm.update(recursive=False, dry_run=True, progress=prog) + assert not sm.module_exists() + sm.update(recursive=False) assert sm.module_exists() sm.config_writer().set_value('path', fp) # change path to something with prefix AFTER url change @@ -397,7 +431,7 @@ class TestSubmodule(TestBase): cpathchange = rwrepo.index.commit("changed sm path") # finally we can commit # update puts the module into place - rm.update(recursive=False) + rm.update(recursive=False, progress=prog) sm.set_parent_commit(cpathchange) assert sm.module_exists() @@ -415,7 +449,12 @@ class TestSubmodule(TestBase): nsm.remove(configuration=False, module=True) assert not nsm.module_exists() and nsm.exists() - rm.update(recursive=False) + + # dry-run does nothing + rm.update(recursive=False, dry_run=True, progress=prog) + + # otherwise it will work + rm.update(recursive=False, progress=prog) assert nsm.module_exists() @@ -429,6 +468,10 @@ class TestSubmodule(TestBase): csmremoved = rwrepo.index.commit("Removed submodule") # an update will remove the module + # not in dry_run + rm.update(recursive=False, dry_run=True) + assert os.path.isdir(smp) + rm.update(recursive=False) assert not os.path.isdir(smp) @@ -444,7 +487,11 @@ class TestSubmodule(TestBase): nsm.set_parent_commit(csmpathchange) prev_commit = nsm.module().head.commit - rm.update(recursive=False) + # dry-run does nothing + rm.update(recursive=False, dry_run=True, progress=prog) + assert nsm.module().remotes.origin.url != nsmurl + + rm.update(recursive=False, progress=prog) assert nsm.module().remotes.origin.url == nsmurl # head changed, as the remote url and its commit changed assert prev_commit != nsm.module().head.commit @@ -473,7 +520,12 @@ class TestSubmodule(TestBase): assert nsmmh.ref.tracking_branch() is None # never set it up until now assert not nsmmh.is_detached - rm.update(recursive=False) + #dry run does nothing + rm.update(recursive=False, dry_run=True, progress=prog) + assert nsmmh.ref.tracking_branch() is None + + # the real thing does + rm.update(recursive=False, progress=prog) assert nsmmh.ref.tracking_branch() is not None assert not nsmmh.is_detached @@ -487,7 +539,8 @@ class TestSubmodule(TestBase): # assure we pull locally only nsmc = nsm.children()[0] nsmc.config_writer().set_value('url', async_url) - rm.update(recursive=True) + rm.update(recursive=True, progress=prog, dry_run=True) # just to run the code + rm.update(recursive=True, progress=prog) - assert len(nsm.children()) == 1 and nsmc.module_exists() + assert len(nsm.children()) == 1 and nsmc.module_exists() @@ -22,7 +22,8 @@ from gitdb.util import ( __all__ = ( "stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", - "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists') + "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', + 'RemoteProgress') #{ Utility Methods @@ -77,6 +78,7 @@ def join_path_native(a, *p): def assure_directory_exists(path, is_file=False): """Assure that the directory pointed to by path exists. + :param is_file: If True, path is assumed to be a file and handled correctly. Otherwise it must be a directory :return: True if the directory was created, False if it already existed""" @@ -103,6 +105,125 @@ def get_user_id(): #{ Classes +class RemoteProgress(object): + """ + Handler providing an interface to parse progress information emitted by git-push + and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. + """ + _num_op_codes = 5 + BEGIN, END, COUNTING, COMPRESSING, WRITING = [1 << x for x in range(_num_op_codes)] + STAGE_MASK = BEGIN|END + OP_MASK = ~STAGE_MASK + + __slots__ = ("_cur_line", "_seen_ops") + re_op_absolute = re.compile("(remote: )?([\w\s]+):\s+()(\d+)()(.*)") + re_op_relative = re.compile("(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)") + + def __init__(self): + self._seen_ops = list() + + def _parse_progress_line(self, line): + """Parse progress information from the given line as retrieved by git-push + or git-fetch + + :return: list(line, ...) list of lines that could not be processed""" + # handle + # Counting objects: 4, done. + # Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done. + self._cur_line = line + sub_lines = line.split('\r') + failed_lines = list() + for sline in sub_lines: + # find esacpe characters and cut them away - regex will not work with + # them as they are non-ascii. As git might expect a tty, it will send them + last_valid_index = None + for i,c in enumerate(reversed(sline)): + if ord(c) < 32: + # its a slice index + last_valid_index = -i-1 + # END character was non-ascii + # END for each character in sline + if last_valid_index is not None: + sline = sline[:last_valid_index] + # END cut away invalid part + sline = sline.rstrip() + + cur_count, max_count = None, None + match = self.re_op_relative.match(sline) + if match is None: + match = self.re_op_absolute.match(sline) + + if not match: + self.line_dropped(sline) + failed_lines.append(sline) + continue + # END could not get match + + op_code = 0 + remote, op_name, percent, cur_count, max_count, message = match.groups() + + # get operation id + if op_name == "Counting objects": + op_code |= self.COUNTING + elif op_name == "Compressing objects": + op_code |= self.COMPRESSING + elif op_name == "Writing objects": + op_code |= self.WRITING + else: + raise ValueError("Operation name %r unknown" % op_name) + + # figure out stage + if op_code not in self._seen_ops: + self._seen_ops.append(op_code) + op_code |= self.BEGIN + # END begin opcode + + if message is None: + message = '' + # END message handling + + message = message.strip() + done_token = ', done.' + if message.endswith(done_token): + op_code |= self.END + message = message[:-len(done_token)] + # END end message handling + + self.update(op_code, cur_count, max_count, message) + # END for each sub line + return failed_lines + + def line_dropped(self, line): + """Called whenever a line could not be understood and was therefore dropped.""" + pass + + def update(self, op_code, cur_count, max_count=None, message=''): + """Called whenever the progress changes + + :param op_code: + Integer allowing to be compared against Operation IDs and stage IDs. + + Stage IDs are BEGIN and END. BEGIN will only be set once for each Operation + ID as well as END. It may be that BEGIN and END are set at once in case only + one progress message was emitted due to the speed of the operation. + Between BEGIN and END, none of these flags will be set + + Operation IDs are all held within the OP_MASK. Only one Operation ID will + be active per call. + :param cur_count: Current absolute count of items + + :param max_count: + The maximum count of items we expect. It may be None in case there is + no maximum number of items or if it is (yet) unknown. + + :param message: + In case of the 'WRITING' operation, it contains the amount of bytes + transferred. It may possibly be used for other purposes as well. + + You may read the contents of the current line in self._cur_line""" + pass + + class Actor(object): """Actors hold information about a person acting on the repository. They can be committers and authors or anything with a name and an email as |