diff options
author | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-08-05 16:22:51 +0000 |
---|---|---|
committer | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-08-05 16:22:51 +0000 |
commit | cf46733632c7279a9fd0fe6ce26f9185a4ae82a9 (patch) | |
tree | da27775a2161723ef342e91af41a8b51fedef405 /tools/server-side/svnpubsub | |
parent | bb0ef45f7c46b0ae221b26265ef98a768c33f820 (diff) | |
download | subversion-tarball-master.tar.gz |
subversion-1.9.7HEADsubversion-1.9.7master
Diffstat (limited to 'tools/server-side/svnpubsub')
-rwxr-xr-x | tools/server-side/svnpubsub/commit-hook.py | 43 | ||||
-rw-r--r-- | tools/server-side/svnpubsub/daemonize.py | 193 | ||||
-rwxr-xr-x | tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd | 2 | ||||
-rwxr-xr-x | tools/server-side/svnpubsub/revprop-change-hook.py | 90 | ||||
-rw-r--r-- | tools/server-side/svnpubsub/svnpubsub/client.py | 54 | ||||
-rw-r--r-- | tools/server-side/svnpubsub/svnpubsub/server.py | 114 | ||||
-rw-r--r-- | tools/server-side/svnpubsub/svnpubsub/util.py | 36 | ||||
-rwxr-xr-x | tools/server-side/svnpubsub/svnwcsub.py | 46 | ||||
-rwxr-xr-x | tools/server-side/svnpubsub/watcher.py | 5 |
9 files changed, 429 insertions, 154 deletions
diff --git a/tools/server-side/svnpubsub/commit-hook.py b/tools/server-side/svnpubsub/commit-hook.py index 4a1a3f3..4e6a1cc 100755 --- a/tools/server-side/svnpubsub/commit-hook.py +++ b/tools/server-side/svnpubsub/commit-hook.py @@ -23,7 +23,6 @@ HOST="127.0.0.1" PORT=2069 import sys -import subprocess try: import simplejson as json except ImportError: @@ -31,32 +30,32 @@ except ImportError: import urllib2 -def svncmd(cmd): - return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) +import svnpubsub.util -def svncmd_uuid(repo): - cmd = "%s uuid %s" % (SVNLOOK, repo) - p = svncmd(cmd) - return p.stdout.read().strip() +def svnlook(cmd, **kwargs): + args = [SVNLOOK] + cmd + return svnpubsub.util.check_output(args, **kwargs) -def svncmd_info(repo, revision): - cmd = "%s info -r %s %s" % (SVNLOOK, revision, repo) - p = svncmd(cmd) - data = p.stdout.read().split("\n") +def svnlook_uuid(repo): + cmd = ["uuid", "--", repo] + return svnlook(cmd).strip() + +def svnlook_info(repo, revision): + cmd = ["info", "-r", revision, "--", repo] + data = svnlook(cmd, universal_newlines=True).split("\n") #print data return {'author': data[0].strip(), 'date': data[1].strip(), 'log': "\n".join(data[3:]).strip()} -def svncmd_changed(repo, revision): - cmd = "%s changed -r %s %s" % (SVNLOOK, revision, repo) - p = svncmd(cmd) +def svnlook_changed(repo, revision): + cmd = ["changed", "-r", revision, "--", repo] + lines = svnlook(cmd, universal_newlines=True).split("\n") changed = {} - while True: - line = p.stdout.readline() - if not line: - break + for line in lines: line = line.strip() + if not line: + continue (flags, filename) = (line[0:3], line[4:]) changed[filename] = {'flags': flags} return changed @@ -71,23 +70,23 @@ def do_put(body): def main(repo, revision): revision = revision.lstrip('r') - i = svncmd_info(repo, revision) + i = svnlook_info(repo, revision) data = {'type': 'svn', 'format': 1, 'id': int(revision), 'changed': {}, - 'repository': svncmd_uuid(repo), + 'repository': svnlook_uuid(repo), 'committer': i['author'], 'log': i['log'], 'date': i['date'], } - data['changed'].update(svncmd_changed(repo, revision)) + data['changed'].update(svnlook_changed(repo, revision)) body = json.dumps(data) do_put(body) if __name__ == "__main__": if len(sys.argv) not in (3, 4): sys.stderr.write("invalid args\n") - sys.exit(0) + sys.exit(1) main(*sys.argv[1:3]) diff --git a/tools/server-side/svnpubsub/daemonize.py b/tools/server-side/svnpubsub/daemonize.py index 8b85258..41b1bec 100644 --- a/tools/server-side/svnpubsub/daemonize.py +++ b/tools/server-side/svnpubsub/daemonize.py @@ -24,6 +24,7 @@ import os import signal import sys import time +import multiprocessing # requires Python 2.6 # possible return values from Daemon.daemonize() @@ -50,11 +51,11 @@ class Daemon(object): def daemonize_exit(self): try: result = self.daemonize() - except (ChildFailed, DaemonFailed) as e: + except (ChildFailed, DaemonFailed), e: # duplicate the exit code sys.exit(e.code) except (ChildTerminatedAbnormally, ChildForkFailed, - DaemonTerminatedAbnormally, DaemonForkFailed) as e: + DaemonTerminatedAbnormally, DaemonForkFailed), e: sys.stderr.write('ERROR: %s\n' % e) sys.exit(1) except ChildResumedIncorrectly: @@ -71,29 +72,41 @@ class Daemon(object): # in original process. daemon is up and running. we're done. def daemonize(self): - # fork off a child that can detach itself from this process. - try: - pid = os.fork() - except OSError as e: - raise ChildForkFailed(e.errno, e.strerror) - - if pid > 0: - # we're in the parent. let's wait for the child to finish setting - # things up -- on our exit, we want to ensure the child is accepting - # connections. - cpid, status = os.waitpid(pid, 0) - assert pid == cpid - if os.WIFEXITED(status): - code = os.WEXITSTATUS(status) - if code: - raise ChildFailed(code) - return DAEMON_RUNNING - - # the child did not exit cleanly. - raise ChildTerminatedAbnormally(status) - + ### review error situations. map to backwards compat. ?? + ### be mindful of daemonize_exit(). + ### we should try and raise ChildFailed / ChildTerminatedAbnormally. + ### ref: older revisions. OR: remove exceptions. + + child_is_ready = multiprocessing.Event() + child_completed = multiprocessing.Event() + + p = multiprocessing.Process(target=self._first_child, + args=(child_is_ready, child_completed)) + p.start() + + # Wait for the child to finish setting things up (in case we need + # to communicate with it). It will only exit when ready. + ### use a timeout here! (parameterized, of course) + p.join() + + ### need to propagate errors, to adjust the return codes + if child_completed.is_set(): + ### what was the exit status? + return DAEMON_COMPLETE + if child_is_ready.is_set(): + return DAEMON_RUNNING + + ### how did we get here?! the immediate child should not exit without + ### signalling ready/complete. some kind of error. + return DAEMON_STARTED + + def _first_child(self, child_is_ready, child_completed): # we're in the child. + ### NOTE: the original design was a bit bunk. Exceptions raised from + ### this point are within the child processes. We need to signal the + ### errors to the parent in other ways. + # decouple from the parent process os.chdir('/') os.umask(0) @@ -102,63 +115,86 @@ class Daemon(object): # remember this pid so the second child can signal it. thispid = os.getpid() - # register a signal handler so the SIGUSR1 doesn't stop the process. - # this object will also record whether if got signalled. - daemon_accepting = SignalCatcher(signal.SIGUSR1) - - # if the daemon process exits before sending SIGUSR1, then we need to see - # the problem. trap SIGCHLD with a SignalCatcher. + # if the daemon process exits before signalling readiness, then we + # need to see the problem. trap SIGCHLD with a SignalCatcher. daemon_exit = SignalCatcher(signal.SIGCHLD) # perform the second fork try: pid = os.fork() - except OSError as e: + except OSError, e: + ### this won't make it to the parent process raise DaemonForkFailed(e.errno, e.strerror) if pid > 0: # in the parent. - # we want to wait for the daemon to signal that it has created and - # bound the socket, and is (thus) ready for connections. if the - # daemon improperly exits before serving, we'll see SIGCHLD and the - # .pause will return. - ### we should add a timeout to this. allow an optional parameter to - ### specify the timeout, in case it takes a long time to start up. - signal.pause() + + # Wait for the child to be ready for operation. + while True: + # The readiness event will invariably be signalled early/first. + # If it *doesn't* get signalled because the child has prematurely + # exited, then we will pause 10ms before noticing the exit. The + # pause is acceptable since that is aberrant/unexpected behavior. + ### is there a way to break this wait() on a signal such as SIGCHLD? + ### parameterize this wait, in case the app knows children may + ### fail quickly? + if child_is_ready.wait(timeout=0.010): + # The child signalled readiness. Yay! + break + if daemon_exit.signalled: + # Whoops. The child exited without signalling :-( + break + # Python 2.6 compat: .wait() may exit when set, but return None + if child_is_ready.is_set(): + break + # A simple timeout. The child is taking a while to prepare. Go + # back and wait for readiness. if daemon_exit.signalled: + # Tell the parent that the child has exited. + ### we need to communicate the exit status, if possible. + child_completed.set() + # reap the daemon process, getting its exit code. bubble it up. cpid, status = os.waitpid(pid, 0) assert pid == cpid if os.WIFEXITED(status): code = os.WEXITSTATUS(status) if code: + ### this won't make it to the parent process raise DaemonFailed(code) + ### this return value is ignored return DAEMON_NOT_RUNNING # the daemon did not exit cleanly. + ### this won't make it to the parent process raise DaemonTerminatedAbnormally(status) - if daemon_accepting.signalled: - # the daemon is up and running, so save the pid and return success. - if self.pidfile: - # Be wary of symlink attacks - try: - os.remove(self.pidfile) - except OSError: - pass - fd = os.open(self.pidfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0444) - os.write(fd, '%d\n' % pid) - os.close(fd) - return DAEMON_STARTED - + # child_is_ready got asserted. the daemon is up and running, so + # save the pid and return success. + if self.pidfile: + # Be wary of symlink attacks + try: + os.remove(self.pidfile) + except OSError: + pass + fd = os.open(self.pidfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0444) + os.write(fd, '%d\n' % pid) + os.close(fd) + + ### this return value is ignored + return DAEMON_STARTED + + ### old code. what to do with this? throw ChildResumedIncorrectly + ### or just toss this and the exception. # some other signal popped us out of the pause. the daemon might not # be running. + ### this won't make it to the parent process raise ChildResumedIncorrectly() - # we're a deamon now. get rid of the final remnants of the parent. - # start by restoring default signal handlers + # we're a daemon now. get rid of the final remnants of the parent: + # restore the signal handlers and switch std* to the proper files. signal.signal(signal.SIGUSR1, signal.SIG_DFL) signal.signal(signal.SIGCHLD, signal.SIG_DFL) sys.stdout.flush() @@ -176,30 +212,31 @@ class Daemon(object): so.close() se.close() - # TEST: don't release the parent immediately. the whole parent stack - # should pause along with this sleep. + ### TEST: don't release the parent immediately. the whole parent stack + ### should pause along with this sleep. #time.sleep(10) # everything is set up. call the initialization function. self.setup() - # sleep for one second before signalling. we want to make sure the - # parent has called signal.pause() - ### we should think of a better wait around the race condition. - time.sleep(1) + ### TEST: exit before signalling. + #sys.exit(0) + #sys.exit(1) - # okay. the daemon is ready. signal the parent to tell it we're set. - os.kill(thispid, signal.SIGUSR1) + # the child is now ready for parent/anyone to communicate with it. + child_is_ready.set() # start the daemon now. self.run() # The daemon is shutting down, so toss the pidfile. - try: - os.remove(self.pidfile) - except OSError: - pass + if self.pidfile: + try: + os.remove(self.pidfile) + except OSError: + pass + ### this return value is ignored return DAEMON_COMPLETE def setup(self): @@ -209,6 +246,34 @@ class Daemon(object): raise NotImplementedError +class _Detacher(Daemon): + def __init__(self, target, logfile='/dev/null', pidfile=None, + args=(), kwargs={}): + Daemon.__init__(self, logfile, pidfile) + self.target = target + self.args = args + self.kwargs = kwargs + + def setup(self): + pass + + def run(self): + self.target(*self.args, **self.kwargs) + + +def run_detached(target, *args, **kwargs): + """Simple function to run TARGET as a detached daemon. + + The additional arguments/keywords will be passed along. This function + does not return -- sys.exit() will be called as appropriate. + + (capture SystemExit if logging/reporting is necessary) + ### if needed, a variant of this func could be written to not exit + """ + d = _Detacher(target, args=args, kwargs=kwargs) + d.daemonize_exit() + + class SignalCatcher(object): def __init__(self, signum): self.signalled = False diff --git a/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd b/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd index 71fc8c8..79b5901 100755 --- a/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd +++ b/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd @@ -26,7 +26,7 @@ pidfile="${svnpubsub_pidfile}" export PYTHON_EGG_CACHE="/home/svn/.python-eggs" command="/usr/local/bin/twistd" -command_interpreter="/usr/local/bin/${svnwcsub_cmd_int}" +command_interpreter="/usr/local/bin/${svnpubsub_cmd_int}" command_args="-y /usr/local/svnpubsub/svnpubsub.tac \ --logfile=/var/log/vc/svnpubsub.log \ --pidfile=${pidfile} \ diff --git a/tools/server-side/svnpubsub/revprop-change-hook.py b/tools/server-side/svnpubsub/revprop-change-hook.py new file mode 100755 index 0000000..3aa857b --- /dev/null +++ b/tools/server-side/svnpubsub/revprop-change-hook.py @@ -0,0 +1,90 @@ +#!/usr/local/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SVNLOOK="/usr/local/svn-install/current/bin/svnlook" +#SVNLOOK="/usr/local/bin/svnlook" + +HOST="127.0.0.1" +PORT=2069 + +import sys +try: + import simplejson as json +except ImportError: + import json + +import urllib2 + + +import svnpubsub.util + +def svnlook(cmd, **kwargs): + args = [SVNLOOK] + cmd + return svnpubsub.util.check_output(args, **kwargs) + +def svnlook_uuid(repo): + cmd = ["uuid", "--", repo] + return svnlook(cmd).strip() + +def svnlook_revprop(repo, revision, propname): + cmd = ["propget", "-r", revision, "--revprop", "--", repo, propname] + data = svnlook(cmd) + #print data + return data + +def do_put(body): + opener = urllib2.build_opener(urllib2.HTTPHandler) + request = urllib2.Request("http://%s:%d/metadata" %(HOST, PORT), data=body) + request.add_header('Content-Type', 'application/json') + request.get_method = lambda: 'PUT' + url = opener.open(request) + + +def main(repo, revision, author, propname, action): + revision = revision.lstrip('r') + if action in ('A', 'M'): + new_value = svnlook_revprop(repo, revision, propname) + elif action == 'D': + new_value = None + else: + sys.stderr.write('Unknown revprop change action "%s"\n' % action) + sys.exit(1) + if action in ('D', 'M'): + old_value = sys.stdin.read() + else: + old_value = None + data = {'type': 'svn', + 'format': 1, + 'id': int(revision), + 'repository': svnlook_uuid(repo), + 'revprop': { + 'name': propname, + 'committer': author, + 'value': new_value, + 'old_value': old_value, + } + } + body = json.dumps(data) + do_put(body) + +if __name__ == "__main__": + if len(sys.argv) != 6: + sys.stderr.write("invalid args\n") + sys.exit(1) + + main(*sys.argv[1:6]) diff --git a/tools/server-side/svnpubsub/svnpubsub/client.py b/tools/server-side/svnpubsub/svnpubsub/client.py index c1631d6..871a5e9 100644 --- a/tools/server-side/svnpubsub/svnpubsub/client.py +++ b/tools/server-side/svnpubsub/svnpubsub/client.py @@ -62,7 +62,8 @@ class SvnpubsubClientException(Exception): class Client(asynchat.async_chat): - def __init__(self, url, commit_callback, event_callback): + def __init__(self, url, commit_callback, event_callback, + metadata_callback = None): asynchat.async_chat.__init__(self) self.last_activity = time.time() @@ -82,7 +83,8 @@ class Client(asynchat.async_chat): self.event_callback = event_callback - self.parser = JSONRecordHandler(commit_callback, event_callback) + self.parser = JSONRecordHandler(commit_callback, event_callback, + metadata_callback) # Wait for the end of headers. Then we start parsing JSON. self.set_terminator(b'\r\n\r\n') @@ -126,36 +128,50 @@ class Client(asynchat.async_chat): self.ibuffer.append(data) +class Notification(object): + def __init__(self, data): + self.__dict__.update(data) + +class Commit(Notification): + KIND = 'COMMIT' + +class Metadata(Notification): + KIND = 'METADATA' + + class JSONRecordHandler: - def __init__(self, commit_callback, event_callback): + def __init__(self, commit_callback, event_callback, metadata_callback): self.commit_callback = commit_callback self.event_callback = event_callback + self.metadata_callback = metadata_callback + + EXPECTED_VERSION = 1 def feed(self, record): obj = json.loads(record) if 'svnpubsub' in obj: actual_version = obj['svnpubsub'].get('version') - EXPECTED_VERSION = 1 - if actual_version != EXPECTED_VERSION: - raise SvnpubsubClientException("Unknown svnpubsub format: %r != %d" - % (actual_format, expected_format)) + if actual_version != self.EXPECTED_VERSION: + raise SvnpubsubClientException( + "Unknown svnpubsub format: %r != %d" + % (actual_version, self.EXPECTED_VERSION)) self.event_callback('version', obj['svnpubsub']['version']) elif 'commit' in obj: commit = Commit(obj['commit']) self.commit_callback(commit) elif 'stillalive' in obj: self.event_callback('ping', obj['stillalive']) - - -class Commit(object): - def __init__(self, commit): - self.__dict__.update(commit) + elif 'metadata' in obj and self.metadata_callback: + metadata = Metadata(obj['metadata']) + self.metadata_callback(metadata) class MultiClient(object): - def __init__(self, urls, commit_callback, event_callback): + def __init__(self, urls, commit_callback, event_callback, + metadata_callback = None): self.commit_callback = commit_callback self.event_callback = event_callback + self.metadata_callback = metadata_callback # No target time, as no work to do self.target_time = 0 @@ -185,9 +201,15 @@ class MultiClient(object): def _add_channel(self, url): # Simply instantiating the client will install it into the global map # for processing in the main event loop. - Client(url, - functools.partial(self.commit_callback, url), - functools.partial(self._reconnect, url)) + if self.metadata_callback: + Client(url, + functools.partial(self.commit_callback, url), + functools.partial(self._reconnect, url), + functools.partial(self.metadata_callback, url)) + else: + Client(url, + functools.partial(self.commit_callback, url), + functools.partial(self._reconnect, url)) def _check_stale(self): now = time.time() diff --git a/tools/server-side/svnpubsub/svnpubsub/server.py b/tools/server-side/svnpubsub/svnpubsub/server.py index faee423..d0cdff9 100644 --- a/tools/server-side/svnpubsub/svnpubsub/server.py +++ b/tools/server-side/svnpubsub/svnpubsub/server.py @@ -25,20 +25,27 @@ # Instead of using a complicated XMPP/AMPQ/JMS/super messaging service, # we have simple HTTP GETs and PUTs to get data in and out. # -# Currently supports both XML and JSON serialization. +# Currently supports JSON serialization. # # Example Sub clients: -# curl -sN http://127.0.0.1:2069/commits -# curl -sN http://127.0.0.1:2069/commits/svn/* -# curl -sN http://127.0.0.1:2069/commits/svn -# curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68 -# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68 +# curl -sN http://127.0.0.1:2069/commits +# curl -sN 'http://127.0.0.1:2069/commits/svn/*' +# curl -sN http://127.0.0.1:2069/commits/svn +# curl -sN 'http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68' +# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68 # -# URL is built into 2 parts: -# /commits/${optional_type}/${optional_repository} +# curl -sN http://127.0.0.1:2069/metadata +# curl -sN 'http://127.0.0.1:2069/metadata/svn/*' +# curl -sN http://127.0.0.1:2069/metadata/svn +# curl -sN 'http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68' +# curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68 # -# If the type is included in the URL, you will only get commits of that type. -# The type can be * and then you will receive commits of any type. +# URLs are constructed from 3 parts: +# /${notification}/${optional_type}/${optional_repository} +# +# Notifications can be sent for commits or metadata (e.g., revprop) changes. +# If the type is included in the URL, you will only get notifications of that type. +# The type can be * and then you will receive notifications of any type. # # If the repository is included in the URL, you will only receive # messages about that repository. The repository can be * and then you @@ -71,7 +78,7 @@ from twisted.python import log import time -class Commit: +class Notification(object): def __init__(self, r): self.__dict__.update(r) if not self.check_value('repository'): @@ -86,7 +93,16 @@ class Commit: def check_value(self, k): return hasattr(self, k) and self.__dict__[k] - def render_commit(self): + def render(self): + raise NotImplementedError + + def render_log(self): + raise NotImplementedError + +class Commit(Notification): + KIND = 'COMMIT' + + def render(self): obj = {'commit': {}} obj['commit'].update(self.__dict__) return json.dumps(obj) @@ -96,20 +112,32 @@ class Commit: paths_changed = " %d paths changed" % len(self.changed) except: paths_changed = "" - return "%s:%s repo '%s' id '%s'%s" % (self.type, - self.format, - self.repository, - self.id, - paths_changed) + return "commit %s:%s repo '%s' id '%s'%s" % ( + self.type, self.format, self.repository, self.id, + paths_changed) + +class Metadata(Notification): + KIND = 'METADATA' + + def render(self): + obj = {'metadata': {}} + obj['metadata'].update(self.__dict__) + return json.dumps(obj) + + def render_log(self): + return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % ( + self.type, self.format, self.repository, self.id, + self.revprop['name']) HEARTBEAT_TIME = 15 class Client(object): - def __init__(self, pubsub, r, type, repository): + def __init__(self, pubsub, r, kind, type, repository): self.pubsub = pubsub r.notifyFinish().addErrback(self.finished) self.r = r + self.kind = kind self.type = type self.repository = repository self.alive = True @@ -123,11 +151,14 @@ class Client(object): except ValueError: pass - def interested_in(self, commit): - if self.type and self.type != commit.type: + def interested_in(self, notification): + if self.kind != notification.KIND: + return False + + if self.type and self.type != notification.type: return False - if self.repository and self.repository != commit.repository: + if self.repository and self.repository != notification.repository: return False return True @@ -164,6 +195,13 @@ class SvnPubSub(resource.Resource): isLeaf = True clients = [] + __notification_uri_map = {'commits': Commit.KIND, + 'metadata': Metadata.KIND} + + def __init__(self, notification_class): + resource.Resource.__init__(self) + self.__notification_class = notification_class + def cc(self): return len(self.clients) @@ -183,6 +221,11 @@ class SvnPubSub(resource.Resource): request.setResponseCode(400) return "Invalid path\n" + kind = self.__notification_uri_map.get(uri[1], None) + if kind is None: + request.setResponseCode(400) + return "Invalid path\n" + if uri_len >= 3: type = uri[2] @@ -195,17 +238,18 @@ class SvnPubSub(resource.Resource): if repository == '*': repository = None - c = Client(self, request, type, repository) + c = Client(self, request, kind, type, repository) self.clients.append(c) c.start() return twisted.web.server.NOT_DONE_YET - def notifyAll(self, commit): - data = commit.render_commit() + def notifyAll(self, notification): + data = notification.render() - log.msg("COMMIT: %s (%d clients)" % (commit.render_log(), self.cc())) + log.msg("%s: %s (%d clients)" + % (notification.KIND, notification.render_log(), self.cc())) for client in self.clients: - if client.interested_in(commit): + if client.interested_in(notification): client.write_data(data) def render_PUT(self, request): @@ -218,19 +262,23 @@ class SvnPubSub(resource.Resource): #import pdb;pdb.set_trace() #print "input: %s" % (input) try: - c = json.loads(input) - commit = Commit(c) + data = json.loads(input) + notification = self.__notification_class(data) except ValueError as e: request.setResponseCode(400) - log.msg("COMMIT: failed due to: %s" % str(e)) - return str(e) - self.notifyAll(commit) + errstr = str(e) + log.msg("%s: failed due to: %s" % (notification.KIND, errstr)) + return errstr + self.notifyAll(notification) return "Ok" + def svnpubsub_server(): root = resource.Resource() - s = SvnPubSub() - root.putChild("commits", s) + c = SvnPubSub(Commit) + m = SvnPubSub(Metadata) + root.putChild('commits', c) + root.putChild('metadata', m) return server.Site(root) if __name__ == "__main__": diff --git a/tools/server-side/svnpubsub/svnpubsub/util.py b/tools/server-side/svnpubsub/svnpubsub/util.py new file mode 100644 index 0000000..e254f8b --- /dev/null +++ b/tools/server-side/svnpubsub/svnpubsub/util.py @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import subprocess as __subprocess + +# check_output() is only available in Python 2.7. Allow us to run with +# earlier versions +try: + __check_output = __subprocess.check_output + def check_output(args, env=None, universal_newlines=False): + return __check_output(args, shell=False, env=env, + universal_newlines=universal_newlines) +except AttributeError: + def check_output(args, env=None, universal_newlines=False): + # note: we only use these three args + pipe = __subprocess.Popen(args, shell=False, env=env, + stdout=__subprocess.PIPE, + universal_newlines=universal_newlines) + output, _ = pipe.communicate() + if pipe.returncode: + raise subprocess.CalledProcessError(pipe.returncode, args) + return output diff --git a/tools/server-side/svnpubsub/svnwcsub.py b/tools/server-side/svnpubsub/svnwcsub.py index 366df7c..8105d87 100755 --- a/tools/server-side/svnpubsub/svnwcsub.py +++ b/tools/server-side/svnpubsub/svnwcsub.py @@ -69,27 +69,22 @@ except ImportError: import daemonize import svnpubsub.client - -# check_output() is only available in Python 2.7. Allow us to run with -# earlier versions -try: - check_output = subprocess.check_output -except AttributeError: - def check_output(args, env): # note: we only use these two args - pipe = subprocess.Popen(args, stdout=subprocess.PIPE, env=env) - output, _ = pipe.communicate() - if pipe.returncode: - raise subprocess.CalledProcessError(pipe.returncode, args) - return output +import svnpubsub.util assert hasattr(subprocess, 'check_call') def check_call(*args, **kwds): - """Wrapper around subprocess.check_call() that logs stderr upon failure.""" + """Wrapper around subprocess.check_call() that logs stderr upon failure, + with an optional list of exit codes to consider non-failure.""" assert 'stderr' not in kwds + if '__okayexits' in kwds: + __okayexits = kwds['__okayexits'] + del kwds['__okayexits'] + else: + __okayexits = set([0]) # EXIT_SUCCESS kwds.update(stderr=subprocess.PIPE) pipe = subprocess.Popen(*args, **kwds) output, errput = pipe.communicate() - if pipe.returncode: + if pipe.returncode not in __okayexits: cmd = args[0] if len(args) else kwds.get('args', '(no command)') # TODO: log stdout too? logging.error('Command failed: returncode=%d command=%r stderr=%r', @@ -103,7 +98,7 @@ def check_call(*args, **kwds): def svn_info(svnbin, env, path): "Run 'svn info' on the target path, returning a dict of info data." args = [svnbin, "info", "--non-interactive", "--", path] - output = check_output(args, env=env).strip() + output = svnpubsub.util.check_output(args, env=env).strip() info = { } for line in output.split('\n'): idx = line.index(':') @@ -303,6 +298,21 @@ class BackgroundWorker(threading.Thread): logging.info("updating: %s", wc.path) + ## Run the hook + HEAD = svn_info(self.svnbin, self.env, wc.url)['Revision'] + if self.hook: + hook_mode = ['pre-update', 'pre-boot'][boot] + logging.info('running hook: %s at %s', + wc.path, hook_mode) + args = [self.hook, hook_mode, wc.path, HEAD, wc.url] + rc = check_call(args, env=self.env, __okayexits=[0, 1]) + if rc == 1: + # TODO: log stderr + logging.warn('hook denied update of %s at %s', + wc.path, hook_mode) + return + del rc + ### we need to move some of these args into the config. these are ### still specific to the ASF setup. args = [self.svnbin, 'switch', @@ -313,12 +323,13 @@ class BackgroundWorker(threading.Thread): '--config-option', 'config:miscellany:use-commit-times=on', '--', - wc.url, + wc.url + '@' + HEAD, wc.path] check_call(args, env=self.env) ### check the loglevel before running 'svn info'? info = svn_info(self.svnbin, self.env, wc.path) + assert info['Revision'] == HEAD logging.info("updated: %s now at r%s", wc.path, info['Revision']) ## Run the hook @@ -533,7 +544,8 @@ def main(args): # We manage the logfile ourselves (along with possible rotation). The # daemon process can just drop stdout/stderr into /dev/null. - d = Daemon('/dev/null', options.pidfile, options.umask, bdec) + d = Daemon('/dev/null', os.path.abspath(options.pidfile), + options.umask, bdec) if options.daemon: # Daemonize the process and call sys.exit() with appropriate code d.daemonize_exit() diff --git a/tools/server-side/svnpubsub/watcher.py b/tools/server-side/svnpubsub/watcher.py index 340b100..11bf066 100755 --- a/tools/server-side/svnpubsub/watcher.py +++ b/tools/server-side/svnpubsub/watcher.py @@ -35,6 +35,9 @@ def _commit(url, commit): print('COMMIT: from %s' % url) pprint.pprint(vars(commit), indent=2) +def _metadata(url, metadata): + print('METADATA: from %s' % url) + pprint.pprint(vars(metadata), indent=2) def _event(url, event_name, event_arg): if event_arg: @@ -44,7 +47,7 @@ def _event(url, event_name, event_arg): def main(urls): - mc = svnpubsub.client.MultiClient(urls, _commit, _event) + mc = svnpubsub.client.MultiClient(urls, _commit, _event, _metadata) mc.run_forever() |