From 7aeaa2fc0abbf439534769e15b3a59a5814cc3d1 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Mon, 29 Mar 2010 11:48:28 -0500 Subject: remote-helpers: add testgit helper Currently the remote helper infrastructure is only used by the curl helper, which does not give a good impression of how remote helpers can be used to interact with foreign repositories. Since implementing such a helper is non-trivial it would be good to have at least one easy-to-follow example demonstrating how to implement a helper that interacts with a foreign vcs using fast-import/fast-export. The testgit helper can be used to interact with remote git repositories by prefixing the url with "testgit::". Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 git-remote-testgit.py (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py new file mode 100644 index 0000000000..f61624e482 --- /dev/null +++ b/git-remote-testgit.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python + +import hashlib +import sys + +from git_remote_helpers.util import die, debug, warn +from git_remote_helpers.git.repo import GitRepo +from git_remote_helpers.git.exporter import GitExporter +from git_remote_helpers.git.importer import GitImporter +from git_remote_helpers.git.non_local import NonLocalGit + +def get_repo(alias, url): + """Returns a git repository object initialized for usage. + """ + + repo = GitRepo(url) + repo.get_revs() + repo.get_head() + + hasher = hashlib.sha1() + hasher.update(repo.path) + repo.hash = hasher.hexdigest() + + repo.get_base_path = lambda base: os.path.join( + base, 'info', 'fast-import', repo.hash) + + prefix = 'refs/testgit/%s/' % alias + debug("prefix: '%s'", prefix) + + repo.gitdir = "" + repo.alias = alias + repo.prefix = prefix + + repo.exporter = GitExporter(repo) + repo.importer = GitImporter(repo) + repo.non_local = NonLocalGit(repo) + + return repo + + +def local_repo(repo, path): + """Returns a git repository object initalized for usage. + """ + + local = GitRepo(path) + + local.non_local = None + local.gitdir = repo.gitdir + local.alias = repo.alias + local.prefix = repo.prefix + local.hash = repo.hash + local.get_base_path = repo.get_base_path + local.exporter = GitExporter(local) + local.importer = GitImporter(local) + + return local + + +def do_capabilities(repo, args): + """Prints the supported capabilities. + """ + + print "import" + print "export" + print "gitdir" + print "refspec refs/heads/*:%s*" % repo.prefix + + print # end capabilities + + +def do_list(repo, args): + """Lists all known references. + + Bug: This will always set the remote head to master for non-local + repositories, since we have no way of determining what the remote + head is at clone time. + """ + + for ref in repo.revs: + debug("? refs/heads/%s", ref) + print "? refs/heads/%s" % ref + + if repo.head: + debug("@refs/heads/%s HEAD" % repo.head) + print "@refs/heads/%s HEAD" % repo.head + else: + debug("@refs/heads/master HEAD") + print "@refs/heads/master HEAD" + + print # end list + + +def update_local_repo(repo): + """Updates (or clones) a local repo. + """ + + if repo.local: + return repo + + path = repo.non_local.clone(repo.gitdir) + repo.non_local.update(repo.gitdir) + repo = local_repo(repo, path) + return repo + + +def do_import(repo, args): + """Exports a fast-import stream from testgit for git to import. + """ + + if len(args) != 1: + die("Import needs exactly one ref") + + if not repo.gitdir: + die("Need gitdir to import") + + repo = update_local_repo(repo) + repo.exporter.export_repo(repo.gitdir) + + +def do_export(repo, args): + """Imports a fast-import stream from git to testgit. + """ + + if not repo.gitdir: + die("Need gitdir to export") + + dirname = repo.get_base_path(repo.gitdir) + + if not os.path.exists(dirname): + os.makedirs(dirname) + + path = os.path.join(dirname, 'testgit.marks') + print path + print path if os.path.exists(path) else "" + sys.stdout.flush() + + update_local_repo(repo) + repo.importer.do_import(repo.gitdir) + repo.non_local.push(repo.gitdir) + + +def do_gitdir(repo, args): + """Stores the location of the gitdir. + """ + + if not args: + die("gitdir needs an argument") + + repo.gitdir = ' '.join(args) + + +COMMANDS = { + 'capabilities': do_capabilities, + 'list': do_list, + 'import': do_import, + 'export': do_export, + 'gitdir': do_gitdir, +} + + +def sanitize(value): + """Cleans up the url. + """ + + if value.startswith('testgit::'): + value = value[9:] + + return value + + +def read_one_line(repo): + """Reads and processes one command. + """ + + line = sys.stdin.readline() + + cmdline = line + + if not cmdline: + warn("Unexpected EOF") + return False + + cmdline = cmdline.strip().split() + if not cmdline: + # Blank line means we're about to quit + return False + + cmd = cmdline.pop(0) + debug("Got command '%s' with args '%s'", cmd, ' '.join(cmdline)) + + if cmd not in COMMANDS: + die("Unknown command, %s", cmd) + + func = COMMANDS[cmd] + func(repo, cmdline) + sys.stdout.flush() + + return True + + +def main(args): + """Starts a new remote helper for the specified repository. + """ + + if len(args) != 3: + die("Expecting exactly three arguments.") + sys.exit(1) + + if os.getenv("GIT_DEBUG_TESTGIT"): + import git_remote_helpers.util + git_remote_helpers.util.DEBUG = True + + alias = sanitize(args[1]) + url = sanitize(args[2]) + + if not alias.isalnum(): + warn("non-alnum alias '%s'", alias) + alias = "tmp" + + args[1] = alias + args[2] = url + + repo = get_repo(alias, url) + + debug("Got arguments %s", args[1:]) + + more = True + + while (more): + more = read_one_line(repo) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) -- cgit v1.2.1 From f733f6a0c690a3d5e399c438d4dc3f2307418171 Mon Sep 17 00:00:00 2001 From: Brian Gernhardt Date: Fri, 9 Apr 2010 11:57:45 -0400 Subject: Makefile: Simplify handling of python scripts The sed script intended to add a standard opening to python scripts was non-compatible and overly complex. Simplifying it down to a set of one-liners removes the compatibility issues of newlines. Moving the environment alterations from the Makefile to the python scripts makes also makes the scripts easier to run in-place. Specifically, the new sed script: - Alters the shebang line to use the configured Python. - Alters any os.getenv("GITPYTHONLIB") calls to use @@INSTLIBDIR@@ as the default. This will replace any existing default or add a default if none is provided. - Replaces the @@INSTLIBDIR@@ placeholder with the directory git installs its python libraries to. The last two steps could be combined into a single step, but is left separate in case someone has another need for @@INSTLIBDIR@@ in their script. Suggested-by: Junio C Hamano Signed-off-by: Brian Gernhardt Acked-by: Sverre Rabbelier Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index f61624e482..92539222c5 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -2,6 +2,8 @@ import hashlib import sys +import os +sys.path.insert(0, os.getenv("GITPYTHONLIB",".")) from git_remote_helpers.util import die, debug, warn from git_remote_helpers.git.repo import GitRepo -- cgit v1.2.1 From 23b093ee087e99049585487f59e262a0e0662b6e Mon Sep 17 00:00:00 2001 From: Brandon Casey Date: Wed, 9 Jun 2010 19:24:54 -0500 Subject: Remove python 2.5'isms The following python 2.5 features were worked around: * the sha module is used as a fallback when the hashlib module is not available * the 'any' built-in method was replaced with a 'for' loop * a conditional expression was replaced with an 'if' statement * the subprocess.check_call method was replaced by a call to subprocess.Popen followed by a call to subprocess.wait with a check of its return status These changes allow the python infrastructure to be used with python 2.4 which is distributed with RedHat's RHEL 5, for example. t5800 was updated to check for python >= 2.4 to reflect these changes. Signed-off-by: Brandon Casey Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index 92539222c5..df9d512f1a 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -1,6 +1,12 @@ #!/usr/bin/env python -import hashlib +# hashlib is only available in python >= 2.5 +try: + import hashlib + _digest = hashlib.sha1 +except ImportError: + import sha + _digest = sha.new import sys import os sys.path.insert(0, os.getenv("GITPYTHONLIB",".")) @@ -19,7 +25,7 @@ def get_repo(alias, url): repo.get_revs() repo.get_head() - hasher = hashlib.sha1() + hasher = _digest() hasher.update(repo.path) repo.hash = hasher.hexdigest() @@ -133,7 +139,10 @@ def do_export(repo, args): path = os.path.join(dirname, 'testgit.marks') print path - print path if os.path.exists(path) else "" + if os.path.exists(path): + print path + else: + print "" sys.stdout.flush() update_local_repo(repo) -- cgit v1.2.1 From 4e51ba238fb92ad732b4d34200fc8f53e29b333f Mon Sep 17 00:00:00 2001 From: Jeff King Date: Sat, 16 Jul 2011 15:03:25 +0200 Subject: git-remote-testgit: import non-HEAD refs Upon receiving an "import" command, the testgit remote helper would ignore the ref asked for by git and generate a fast-export stream based on HEAD. Instead, we should actually give git the ref it asked for. This requires adding a new parameter to the export_repo method in the remote-helpers python library, which may be used by code outside of git.git. We use a default parameter so that callers without the new parameter will get the same behavior as before. Signed-off-by: Jeff King Signed-off-by: Sverre Rabbelier Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index df9d512f1a..e4a99a33ef 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -122,7 +122,7 @@ def do_import(repo, args): die("Need gitdir to import") repo = update_local_repo(repo) - repo.exporter.export_repo(repo.gitdir) + repo.exporter.export_repo(repo.gitdir, args) def do_export(repo, args): -- cgit v1.2.1 From e173587252ea0db16efc5c64c2cb165ccb406495 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Sat, 16 Jul 2011 15:03:28 +0200 Subject: remote-helpers: export GIT_DIR variable to helpers The gitdir capability is recognized by git and can be used to tell the helper where the .git directory is. But it is not mentioned in the documentation and considered worse than if gitdir was passed via GIT_DIR environment variable. Remove support for the gitdir capability and export GIT_DIR instead. Teach testgit to use env instead of the now-removed gitdir command. [sr: fixed up documentation] Signed-off-by: Dmitry Ivankov Signed-off-by: Sverre Rabbelier Acked-by: Jeff King Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index e4a99a33ef..b0c1e9b273 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -35,7 +35,7 @@ def get_repo(alias, url): prefix = 'refs/testgit/%s/' % alias debug("prefix: '%s'", prefix) - repo.gitdir = "" + repo.gitdir = os.environ["GIT_DIR"] repo.alias = alias repo.prefix = prefix @@ -70,7 +70,6 @@ def do_capabilities(repo, args): print "import" print "export" - print "gitdir" print "refspec refs/heads/*:%s*" % repo.prefix print # end capabilities @@ -150,22 +149,11 @@ def do_export(repo, args): repo.non_local.push(repo.gitdir) -def do_gitdir(repo, args): - """Stores the location of the gitdir. - """ - - if not args: - die("gitdir needs an argument") - - repo.gitdir = ' '.join(args) - - COMMANDS = { 'capabilities': do_capabilities, 'list': do_list, 'import': do_import, 'export': do_export, - 'gitdir': do_gitdir, } -- cgit v1.2.1 From 0fb56ce716090248ed4895aff69dd3953b00882f Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 16 Jul 2011 15:03:30 +0200 Subject: git-remote-testgit: only push for non-local repositories Trying to push for local repositories will fail since there is no local checkout in .git/info/... to push from as that is only used for non-local repositories (local repositories are pushed to directly). This went unnoticed because the transport helper infrastructure does not check the return value of the helper. Signed-off-by: Sverre Rabbelier Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index b0c1e9b273..cdbc49495f 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -146,7 +146,9 @@ def do_export(repo, args): update_local_repo(repo) repo.importer.do_import(repo.gitdir) - repo.non_local.push(repo.gitdir) + + if not repo.local: + repo.non_local.push(repo.gitdir) COMMANDS = { -- cgit v1.2.1 From 1f25c50419c5f46cd6b818438fe641cf942ee6ad Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 16 Jul 2011 15:03:36 +0200 Subject: transport-helper: use the new done feature where possible In other words, use fast-export --use-done-feature to add a 'done' command at the end of streams passed to remote helpers' "import" commands, and teach the remote helpers implementing "export" to use the 'done' command in turn when producing their streams. The trailing \n in the protocol signals the helper that the connection is about to close, allowing it to do whatever cleanup neccesary. Previously, the connection would already be closed by the time the trailing \n was to be written. Now that the remote-helper protocol uses the new done command in its fast-import streams, this is no longer the case and we can safely write the trailing \n. Signed-off-by: Sverre Rabbelier Acked-by: Jeff King Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index cdbc49495f..af4d040353 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -123,6 +123,8 @@ def do_import(repo, args): repo = update_local_repo(repo) repo.exporter.export_repo(repo.gitdir, args) + print "done" + def do_export(repo, args): """Imports a fast-import stream from git to testgit. -- cgit v1.2.1 From 6c8151a32e59c3109b3acc886358bfe6c14612fb Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 16 Jul 2011 15:03:37 +0200 Subject: transport-helper: update ref status after push with export Also add check_output from python 2.7. Signed-off-by: Sverre Rabbelier Acked-by: Jeff King Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index af4d040353..0b5928d290 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -147,11 +147,15 @@ def do_export(repo, args): sys.stdout.flush() update_local_repo(repo) - repo.importer.do_import(repo.gitdir) + changed = repo.importer.do_import(repo.gitdir) if not repo.local: repo.non_local.push(repo.gitdir) + for ref in changed: + print "ok %s" % ref + print + COMMANDS = { 'capabilities': do_capabilities, -- cgit v1.2.1 From 9504bc9d5a1e672ce5945679f86294e61bbea3a6 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 16 Jul 2011 15:03:38 +0200 Subject: transport-helper: change import semantics Currently the helper must somehow guess how many import statements to read before it starts outputting its fast-export stream. This is because the remote helper infrastructure runs fast-import only once, so the helper is forced to output one stream for all import commands it will receive. The only reason this worked in the past was because only one ref was imported at a time. Change the semantics of the import statement such that it matches that of the push statement. That is, the import statement is followed by a series of import statements that are terminated by a '\n'. Signed-off-by: Sverre Rabbelier Acked-by: Jeff King Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index 0b5928d290..1ed7a5651e 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -120,8 +120,22 @@ def do_import(repo, args): if not repo.gitdir: die("Need gitdir to import") + ref = args[0] + refs = [ref] + + while True: + line = sys.stdin.readline() + if line == '\n': + break + if not line.startswith('import '): + die("Expected import line.") + + # strip of leading 'import ' + ref = line[7:].strip() + refs.append(ref) + repo = update_local_repo(repo) - repo.exporter.export_repo(repo.gitdir, args) + repo.exporter.export_repo(repo.gitdir, refs) print "done" -- cgit v1.2.1 From a515ebe9f1ac9bc248c12a291dc008570de505ca Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 16 Jul 2011 15:03:40 +0200 Subject: transport-helper: implement marks location as capability Now that the gitdir location is exported as an environment variable this can be implemented elegantly without requiring any explicit flushes nor an ad-hoc exchange of values. Signed-off-by: Sverre Rabbelier Acked-by: Jeff King Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index 1ed7a5651e..e9c832bfd3 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -72,6 +72,17 @@ def do_capabilities(repo, args): print "export" print "refspec refs/heads/*:%s*" % repo.prefix + dirname = repo.get_base_path(repo.gitdir) + + if not os.path.exists(dirname): + os.makedirs(dirname) + + path = os.path.join(dirname, 'testgit.marks') + + print "*export-marks %s" % path + if os.path.exists(path): + print "*import-marks %s" % path + print # end capabilities @@ -147,19 +158,6 @@ def do_export(repo, args): if not repo.gitdir: die("Need gitdir to export") - dirname = repo.get_base_path(repo.gitdir) - - if not os.path.exists(dirname): - os.makedirs(dirname) - - path = os.path.join(dirname, 'testgit.marks') - print path - if os.path.exists(path): - print path - else: - print "" - sys.stdout.flush() - update_local_repo(repo) changed = repo.importer.do_import(repo.gitdir) -- cgit v1.2.1 From 9609dc9ddcb1a79112eb45c88244cdbb622bd5bc Mon Sep 17 00:00:00 2001 From: Matthieu Moy Date: Thu, 1 Sep 2011 18:49:38 +0200 Subject: (short) documentation for the testgit remote helper While it's not a command meant to be used by actual users (hence, not mentionned in git(1)), this command is a very precious help for remote-helpers authors. The best place for such technical doc is the source code, but users may not find it without a link in a manpage. Signed-off-by: Matthieu Moy Acked-by: Sverre Rabbelier Signed-off-by: Junio C Hamano --- git-remote-testgit.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'git-remote-testgit.py') diff --git a/git-remote-testgit.py b/git-remote-testgit.py index e9c832bfd3..3dc4851cfc 100644 --- a/git-remote-testgit.py +++ b/git-remote-testgit.py @@ -1,5 +1,18 @@ #!/usr/bin/env python +# This command is a simple remote-helper, that is used both as a +# testcase for the remote-helper functionality, and as an example to +# show remote-helper authors one possible implementation. +# +# This is a Git <-> Git importer/exporter, that simply uses git +# fast-import and git fast-export to consume and produce fast-import +# streams. +# +# To understand better the way things work, one can activate debug +# traces by setting (to any value) the environment variables +# GIT_TRANSPORT_HELPER_DEBUG and GIT_DEBUG_TESTGIT, to see messages +# from the transport-helper side, or from this example remote-helper. + # hashlib is only available in python >= 2.5 try: import hashlib -- cgit v1.2.1