summaryrefslogtreecommitdiff
path: root/tools/server-side/svnpubsub
diff options
context:
space:
mode:
authorLorry Tar Creator <lorry-tar-importer@lorry>2017-08-05 16:22:51 +0000
committerLorry Tar Creator <lorry-tar-importer@lorry>2017-08-05 16:22:51 +0000
commitcf46733632c7279a9fd0fe6ce26f9185a4ae82a9 (patch)
treeda27775a2161723ef342e91af41a8b51fedef405 /tools/server-side/svnpubsub
parentbb0ef45f7c46b0ae221b26265ef98a768c33f820 (diff)
downloadsubversion-tarball-master.tar.gz
Diffstat (limited to 'tools/server-side/svnpubsub')
-rwxr-xr-xtools/server-side/svnpubsub/commit-hook.py43
-rw-r--r--tools/server-side/svnpubsub/daemonize.py193
-rwxr-xr-xtools/server-side/svnpubsub/rc.d/svnpubsub.freebsd2
-rwxr-xr-xtools/server-side/svnpubsub/revprop-change-hook.py90
-rw-r--r--tools/server-side/svnpubsub/svnpubsub/client.py54
-rw-r--r--tools/server-side/svnpubsub/svnpubsub/server.py114
-rw-r--r--tools/server-side/svnpubsub/svnpubsub/util.py36
-rwxr-xr-xtools/server-side/svnpubsub/svnwcsub.py46
-rwxr-xr-xtools/server-side/svnpubsub/watcher.py5
9 files changed, 429 insertions, 154 deletions
diff --git a/tools/server-side/svnpubsub/commit-hook.py b/tools/server-side/svnpubsub/commit-hook.py
index 4a1a3f3..4e6a1cc 100755
--- a/tools/server-side/svnpubsub/commit-hook.py
+++ b/tools/server-side/svnpubsub/commit-hook.py
@@ -23,7 +23,6 @@ HOST="127.0.0.1"
PORT=2069
import sys
-import subprocess
try:
import simplejson as json
except ImportError:
@@ -31,32 +30,32 @@ except ImportError:
import urllib2
-def svncmd(cmd):
- return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+import svnpubsub.util
-def svncmd_uuid(repo):
- cmd = "%s uuid %s" % (SVNLOOK, repo)
- p = svncmd(cmd)
- return p.stdout.read().strip()
+def svnlook(cmd, **kwargs):
+ args = [SVNLOOK] + cmd
+ return svnpubsub.util.check_output(args, **kwargs)
-def svncmd_info(repo, revision):
- cmd = "%s info -r %s %s" % (SVNLOOK, revision, repo)
- p = svncmd(cmd)
- data = p.stdout.read().split("\n")
+def svnlook_uuid(repo):
+ cmd = ["uuid", "--", repo]
+ return svnlook(cmd).strip()
+
+def svnlook_info(repo, revision):
+ cmd = ["info", "-r", revision, "--", repo]
+ data = svnlook(cmd, universal_newlines=True).split("\n")
#print data
return {'author': data[0].strip(),
'date': data[1].strip(),
'log': "\n".join(data[3:]).strip()}
-def svncmd_changed(repo, revision):
- cmd = "%s changed -r %s %s" % (SVNLOOK, revision, repo)
- p = svncmd(cmd)
+def svnlook_changed(repo, revision):
+ cmd = ["changed", "-r", revision, "--", repo]
+ lines = svnlook(cmd, universal_newlines=True).split("\n")
changed = {}
- while True:
- line = p.stdout.readline()
- if not line:
- break
+ for line in lines:
line = line.strip()
+ if not line:
+ continue
(flags, filename) = (line[0:3], line[4:])
changed[filename] = {'flags': flags}
return changed
@@ -71,23 +70,23 @@ def do_put(body):
def main(repo, revision):
revision = revision.lstrip('r')
- i = svncmd_info(repo, revision)
+ i = svnlook_info(repo, revision)
data = {'type': 'svn',
'format': 1,
'id': int(revision),
'changed': {},
- 'repository': svncmd_uuid(repo),
+ 'repository': svnlook_uuid(repo),
'committer': i['author'],
'log': i['log'],
'date': i['date'],
}
- data['changed'].update(svncmd_changed(repo, revision))
+ data['changed'].update(svnlook_changed(repo, revision))
body = json.dumps(data)
do_put(body)
if __name__ == "__main__":
if len(sys.argv) not in (3, 4):
sys.stderr.write("invalid args\n")
- sys.exit(0)
+ sys.exit(1)
main(*sys.argv[1:3])
diff --git a/tools/server-side/svnpubsub/daemonize.py b/tools/server-side/svnpubsub/daemonize.py
index 8b85258..41b1bec 100644
--- a/tools/server-side/svnpubsub/daemonize.py
+++ b/tools/server-side/svnpubsub/daemonize.py
@@ -24,6 +24,7 @@ import os
import signal
import sys
import time
+import multiprocessing # requires Python 2.6
# possible return values from Daemon.daemonize()
@@ -50,11 +51,11 @@ class Daemon(object):
def daemonize_exit(self):
try:
result = self.daemonize()
- except (ChildFailed, DaemonFailed) as e:
+ except (ChildFailed, DaemonFailed), e:
# duplicate the exit code
sys.exit(e.code)
except (ChildTerminatedAbnormally, ChildForkFailed,
- DaemonTerminatedAbnormally, DaemonForkFailed) as e:
+ DaemonTerminatedAbnormally, DaemonForkFailed), e:
sys.stderr.write('ERROR: %s\n' % e)
sys.exit(1)
except ChildResumedIncorrectly:
@@ -71,29 +72,41 @@ class Daemon(object):
# in original process. daemon is up and running. we're done.
def daemonize(self):
- # fork off a child that can detach itself from this process.
- try:
- pid = os.fork()
- except OSError as e:
- raise ChildForkFailed(e.errno, e.strerror)
-
- if pid > 0:
- # we're in the parent. let's wait for the child to finish setting
- # things up -- on our exit, we want to ensure the child is accepting
- # connections.
- cpid, status = os.waitpid(pid, 0)
- assert pid == cpid
- if os.WIFEXITED(status):
- code = os.WEXITSTATUS(status)
- if code:
- raise ChildFailed(code)
- return DAEMON_RUNNING
-
- # the child did not exit cleanly.
- raise ChildTerminatedAbnormally(status)
-
+ ### review error situations. map to backwards compat. ??
+ ### be mindful of daemonize_exit().
+ ### we should try and raise ChildFailed / ChildTerminatedAbnormally.
+ ### ref: older revisions. OR: remove exceptions.
+
+ child_is_ready = multiprocessing.Event()
+ child_completed = multiprocessing.Event()
+
+ p = multiprocessing.Process(target=self._first_child,
+ args=(child_is_ready, child_completed))
+ p.start()
+
+ # Wait for the child to finish setting things up (in case we need
+ # to communicate with it). It will only exit when ready.
+ ### use a timeout here! (parameterized, of course)
+ p.join()
+
+ ### need to propagate errors, to adjust the return codes
+ if child_completed.is_set():
+ ### what was the exit status?
+ return DAEMON_COMPLETE
+ if child_is_ready.is_set():
+ return DAEMON_RUNNING
+
+ ### how did we get here?! the immediate child should not exit without
+ ### signalling ready/complete. some kind of error.
+ return DAEMON_STARTED
+
+ def _first_child(self, child_is_ready, child_completed):
# we're in the child.
+ ### NOTE: the original design was a bit bunk. Exceptions raised from
+ ### this point are within the child processes. We need to signal the
+ ### errors to the parent in other ways.
+
# decouple from the parent process
os.chdir('/')
os.umask(0)
@@ -102,63 +115,86 @@ class Daemon(object):
# remember this pid so the second child can signal it.
thispid = os.getpid()
- # register a signal handler so the SIGUSR1 doesn't stop the process.
- # this object will also record whether if got signalled.
- daemon_accepting = SignalCatcher(signal.SIGUSR1)
-
- # if the daemon process exits before sending SIGUSR1, then we need to see
- # the problem. trap SIGCHLD with a SignalCatcher.
+ # if the daemon process exits before signalling readiness, then we
+ # need to see the problem. trap SIGCHLD with a SignalCatcher.
daemon_exit = SignalCatcher(signal.SIGCHLD)
# perform the second fork
try:
pid = os.fork()
- except OSError as e:
+ except OSError, e:
+ ### this won't make it to the parent process
raise DaemonForkFailed(e.errno, e.strerror)
if pid > 0:
# in the parent.
- # we want to wait for the daemon to signal that it has created and
- # bound the socket, and is (thus) ready for connections. if the
- # daemon improperly exits before serving, we'll see SIGCHLD and the
- # .pause will return.
- ### we should add a timeout to this. allow an optional parameter to
- ### specify the timeout, in case it takes a long time to start up.
- signal.pause()
+
+ # Wait for the child to be ready for operation.
+ while True:
+ # The readiness event will invariably be signalled early/first.
+ # If it *doesn't* get signalled because the child has prematurely
+ # exited, then we will pause 10ms before noticing the exit. The
+ # pause is acceptable since that is aberrant/unexpected behavior.
+ ### is there a way to break this wait() on a signal such as SIGCHLD?
+ ### parameterize this wait, in case the app knows children may
+ ### fail quickly?
+ if child_is_ready.wait(timeout=0.010):
+ # The child signalled readiness. Yay!
+ break
+ if daemon_exit.signalled:
+ # Whoops. The child exited without signalling :-(
+ break
+ # Python 2.6 compat: .wait() may exit when set, but return None
+ if child_is_ready.is_set():
+ break
+ # A simple timeout. The child is taking a while to prepare. Go
+ # back and wait for readiness.
if daemon_exit.signalled:
+ # Tell the parent that the child has exited.
+ ### we need to communicate the exit status, if possible.
+ child_completed.set()
+
# reap the daemon process, getting its exit code. bubble it up.
cpid, status = os.waitpid(pid, 0)
assert pid == cpid
if os.WIFEXITED(status):
code = os.WEXITSTATUS(status)
if code:
+ ### this won't make it to the parent process
raise DaemonFailed(code)
+ ### this return value is ignored
return DAEMON_NOT_RUNNING
# the daemon did not exit cleanly.
+ ### this won't make it to the parent process
raise DaemonTerminatedAbnormally(status)
- if daemon_accepting.signalled:
- # the daemon is up and running, so save the pid and return success.
- if self.pidfile:
- # Be wary of symlink attacks
- try:
- os.remove(self.pidfile)
- except OSError:
- pass
- fd = os.open(self.pidfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0444)
- os.write(fd, '%d\n' % pid)
- os.close(fd)
- return DAEMON_STARTED
-
+ # child_is_ready got asserted. the daemon is up and running, so
+ # save the pid and return success.
+ if self.pidfile:
+ # Be wary of symlink attacks
+ try:
+ os.remove(self.pidfile)
+ except OSError:
+ pass
+ fd = os.open(self.pidfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0444)
+ os.write(fd, '%d\n' % pid)
+ os.close(fd)
+
+ ### this return value is ignored
+ return DAEMON_STARTED
+
+ ### old code. what to do with this? throw ChildResumedIncorrectly
+ ### or just toss this and the exception.
# some other signal popped us out of the pause. the daemon might not
# be running.
+ ### this won't make it to the parent process
raise ChildResumedIncorrectly()
- # we're a deamon now. get rid of the final remnants of the parent.
- # start by restoring default signal handlers
+ # we're a daemon now. get rid of the final remnants of the parent:
+ # restore the signal handlers and switch std* to the proper files.
signal.signal(signal.SIGUSR1, signal.SIG_DFL)
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
sys.stdout.flush()
@@ -176,30 +212,31 @@ class Daemon(object):
so.close()
se.close()
- # TEST: don't release the parent immediately. the whole parent stack
- # should pause along with this sleep.
+ ### TEST: don't release the parent immediately. the whole parent stack
+ ### should pause along with this sleep.
#time.sleep(10)
# everything is set up. call the initialization function.
self.setup()
- # sleep for one second before signalling. we want to make sure the
- # parent has called signal.pause()
- ### we should think of a better wait around the race condition.
- time.sleep(1)
+ ### TEST: exit before signalling.
+ #sys.exit(0)
+ #sys.exit(1)
- # okay. the daemon is ready. signal the parent to tell it we're set.
- os.kill(thispid, signal.SIGUSR1)
+ # the child is now ready for parent/anyone to communicate with it.
+ child_is_ready.set()
# start the daemon now.
self.run()
# The daemon is shutting down, so toss the pidfile.
- try:
- os.remove(self.pidfile)
- except OSError:
- pass
+ if self.pidfile:
+ try:
+ os.remove(self.pidfile)
+ except OSError:
+ pass
+ ### this return value is ignored
return DAEMON_COMPLETE
def setup(self):
@@ -209,6 +246,34 @@ class Daemon(object):
raise NotImplementedError
+class _Detacher(Daemon):
+ def __init__(self, target, logfile='/dev/null', pidfile=None,
+ args=(), kwargs={}):
+ Daemon.__init__(self, logfile, pidfile)
+ self.target = target
+ self.args = args
+ self.kwargs = kwargs
+
+ def setup(self):
+ pass
+
+ def run(self):
+ self.target(*self.args, **self.kwargs)
+
+
+def run_detached(target, *args, **kwargs):
+ """Simple function to run TARGET as a detached daemon.
+
+ The additional arguments/keywords will be passed along. This function
+ does not return -- sys.exit() will be called as appropriate.
+
+ (capture SystemExit if logging/reporting is necessary)
+ ### if needed, a variant of this func could be written to not exit
+ """
+ d = _Detacher(target, args=args, kwargs=kwargs)
+ d.daemonize_exit()
+
+
class SignalCatcher(object):
def __init__(self, signum):
self.signalled = False
diff --git a/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd b/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd
index 71fc8c8..79b5901 100755
--- a/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd
+++ b/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd
@@ -26,7 +26,7 @@ pidfile="${svnpubsub_pidfile}"
export PYTHON_EGG_CACHE="/home/svn/.python-eggs"
command="/usr/local/bin/twistd"
-command_interpreter="/usr/local/bin/${svnwcsub_cmd_int}"
+command_interpreter="/usr/local/bin/${svnpubsub_cmd_int}"
command_args="-y /usr/local/svnpubsub/svnpubsub.tac \
--logfile=/var/log/vc/svnpubsub.log \
--pidfile=${pidfile} \
diff --git a/tools/server-side/svnpubsub/revprop-change-hook.py b/tools/server-side/svnpubsub/revprop-change-hook.py
new file mode 100755
index 0000000..3aa857b
--- /dev/null
+++ b/tools/server-side/svnpubsub/revprop-change-hook.py
@@ -0,0 +1,90 @@
+#!/usr/local/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+SVNLOOK="/usr/local/svn-install/current/bin/svnlook"
+#SVNLOOK="/usr/local/bin/svnlook"
+
+HOST="127.0.0.1"
+PORT=2069
+
+import sys
+try:
+ import simplejson as json
+except ImportError:
+ import json
+
+import urllib2
+
+
+import svnpubsub.util
+
+def svnlook(cmd, **kwargs):
+ args = [SVNLOOK] + cmd
+ return svnpubsub.util.check_output(args, **kwargs)
+
+def svnlook_uuid(repo):
+ cmd = ["uuid", "--", repo]
+ return svnlook(cmd).strip()
+
+def svnlook_revprop(repo, revision, propname):
+ cmd = ["propget", "-r", revision, "--revprop", "--", repo, propname]
+ data = svnlook(cmd)
+ #print data
+ return data
+
+def do_put(body):
+ opener = urllib2.build_opener(urllib2.HTTPHandler)
+ request = urllib2.Request("http://%s:%d/metadata" %(HOST, PORT), data=body)
+ request.add_header('Content-Type', 'application/json')
+ request.get_method = lambda: 'PUT'
+ url = opener.open(request)
+
+
+def main(repo, revision, author, propname, action):
+ revision = revision.lstrip('r')
+ if action in ('A', 'M'):
+ new_value = svnlook_revprop(repo, revision, propname)
+ elif action == 'D':
+ new_value = None
+ else:
+ sys.stderr.write('Unknown revprop change action "%s"\n' % action)
+ sys.exit(1)
+ if action in ('D', 'M'):
+ old_value = sys.stdin.read()
+ else:
+ old_value = None
+ data = {'type': 'svn',
+ 'format': 1,
+ 'id': int(revision),
+ 'repository': svnlook_uuid(repo),
+ 'revprop': {
+ 'name': propname,
+ 'committer': author,
+ 'value': new_value,
+ 'old_value': old_value,
+ }
+ }
+ body = json.dumps(data)
+ do_put(body)
+
+if __name__ == "__main__":
+ if len(sys.argv) != 6:
+ sys.stderr.write("invalid args\n")
+ sys.exit(1)
+
+ main(*sys.argv[1:6])
diff --git a/tools/server-side/svnpubsub/svnpubsub/client.py b/tools/server-side/svnpubsub/svnpubsub/client.py
index c1631d6..871a5e9 100644
--- a/tools/server-side/svnpubsub/svnpubsub/client.py
+++ b/tools/server-side/svnpubsub/svnpubsub/client.py
@@ -62,7 +62,8 @@ class SvnpubsubClientException(Exception):
class Client(asynchat.async_chat):
- def __init__(self, url, commit_callback, event_callback):
+ def __init__(self, url, commit_callback, event_callback,
+ metadata_callback = None):
asynchat.async_chat.__init__(self)
self.last_activity = time.time()
@@ -82,7 +83,8 @@ class Client(asynchat.async_chat):
self.event_callback = event_callback
- self.parser = JSONRecordHandler(commit_callback, event_callback)
+ self.parser = JSONRecordHandler(commit_callback, event_callback,
+ metadata_callback)
# Wait for the end of headers. Then we start parsing JSON.
self.set_terminator(b'\r\n\r\n')
@@ -126,36 +128,50 @@ class Client(asynchat.async_chat):
self.ibuffer.append(data)
+class Notification(object):
+ def __init__(self, data):
+ self.__dict__.update(data)
+
+class Commit(Notification):
+ KIND = 'COMMIT'
+
+class Metadata(Notification):
+ KIND = 'METADATA'
+
+
class JSONRecordHandler:
- def __init__(self, commit_callback, event_callback):
+ def __init__(self, commit_callback, event_callback, metadata_callback):
self.commit_callback = commit_callback
self.event_callback = event_callback
+ self.metadata_callback = metadata_callback
+
+ EXPECTED_VERSION = 1
def feed(self, record):
obj = json.loads(record)
if 'svnpubsub' in obj:
actual_version = obj['svnpubsub'].get('version')
- EXPECTED_VERSION = 1
- if actual_version != EXPECTED_VERSION:
- raise SvnpubsubClientException("Unknown svnpubsub format: %r != %d"
- % (actual_format, expected_format))
+ if actual_version != self.EXPECTED_VERSION:
+ raise SvnpubsubClientException(
+ "Unknown svnpubsub format: %r != %d"
+ % (actual_version, self.EXPECTED_VERSION))
self.event_callback('version', obj['svnpubsub']['version'])
elif 'commit' in obj:
commit = Commit(obj['commit'])
self.commit_callback(commit)
elif 'stillalive' in obj:
self.event_callback('ping', obj['stillalive'])
-
-
-class Commit(object):
- def __init__(self, commit):
- self.__dict__.update(commit)
+ elif 'metadata' in obj and self.metadata_callback:
+ metadata = Metadata(obj['metadata'])
+ self.metadata_callback(metadata)
class MultiClient(object):
- def __init__(self, urls, commit_callback, event_callback):
+ def __init__(self, urls, commit_callback, event_callback,
+ metadata_callback = None):
self.commit_callback = commit_callback
self.event_callback = event_callback
+ self.metadata_callback = metadata_callback
# No target time, as no work to do
self.target_time = 0
@@ -185,9 +201,15 @@ class MultiClient(object):
def _add_channel(self, url):
# Simply instantiating the client will install it into the global map
# for processing in the main event loop.
- Client(url,
- functools.partial(self.commit_callback, url),
- functools.partial(self._reconnect, url))
+ if self.metadata_callback:
+ Client(url,
+ functools.partial(self.commit_callback, url),
+ functools.partial(self._reconnect, url),
+ functools.partial(self.metadata_callback, url))
+ else:
+ Client(url,
+ functools.partial(self.commit_callback, url),
+ functools.partial(self._reconnect, url))
def _check_stale(self):
now = time.time()
diff --git a/tools/server-side/svnpubsub/svnpubsub/server.py b/tools/server-side/svnpubsub/svnpubsub/server.py
index faee423..d0cdff9 100644
--- a/tools/server-side/svnpubsub/svnpubsub/server.py
+++ b/tools/server-side/svnpubsub/svnpubsub/server.py
@@ -25,20 +25,27 @@
# Instead of using a complicated XMPP/AMPQ/JMS/super messaging service,
# we have simple HTTP GETs and PUTs to get data in and out.
#
-# Currently supports both XML and JSON serialization.
+# Currently supports JSON serialization.
#
# Example Sub clients:
-# curl -sN http://127.0.0.1:2069/commits
-# curl -sN http://127.0.0.1:2069/commits/svn/*
-# curl -sN http://127.0.0.1:2069/commits/svn
-# curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68
-# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
+# curl -sN http://127.0.0.1:2069/commits
+# curl -sN 'http://127.0.0.1:2069/commits/svn/*'
+# curl -sN http://127.0.0.1:2069/commits/svn
+# curl -sN 'http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68'
+# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
#
-# URL is built into 2 parts:
-# /commits/${optional_type}/${optional_repository}
+# curl -sN http://127.0.0.1:2069/metadata
+# curl -sN 'http://127.0.0.1:2069/metadata/svn/*'
+# curl -sN http://127.0.0.1:2069/metadata/svn
+# curl -sN 'http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68'
+# curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68
#
-# If the type is included in the URL, you will only get commits of that type.
-# The type can be * and then you will receive commits of any type.
+# URLs are constructed from 3 parts:
+# /${notification}/${optional_type}/${optional_repository}
+#
+# Notifications can be sent for commits or metadata (e.g., revprop) changes.
+# If the type is included in the URL, you will only get notifications of that type.
+# The type can be * and then you will receive notifications of any type.
#
# If the repository is included in the URL, you will only receive
# messages about that repository. The repository can be * and then you
@@ -71,7 +78,7 @@ from twisted.python import log
import time
-class Commit:
+class Notification(object):
def __init__(self, r):
self.__dict__.update(r)
if not self.check_value('repository'):
@@ -86,7 +93,16 @@ class Commit:
def check_value(self, k):
return hasattr(self, k) and self.__dict__[k]
- def render_commit(self):
+ def render(self):
+ raise NotImplementedError
+
+ def render_log(self):
+ raise NotImplementedError
+
+class Commit(Notification):
+ KIND = 'COMMIT'
+
+ def render(self):
obj = {'commit': {}}
obj['commit'].update(self.__dict__)
return json.dumps(obj)
@@ -96,20 +112,32 @@ class Commit:
paths_changed = " %d paths changed" % len(self.changed)
except:
paths_changed = ""
- return "%s:%s repo '%s' id '%s'%s" % (self.type,
- self.format,
- self.repository,
- self.id,
- paths_changed)
+ return "commit %s:%s repo '%s' id '%s'%s" % (
+ self.type, self.format, self.repository, self.id,
+ paths_changed)
+
+class Metadata(Notification):
+ KIND = 'METADATA'
+
+ def render(self):
+ obj = {'metadata': {}}
+ obj['metadata'].update(self.__dict__)
+ return json.dumps(obj)
+
+ def render_log(self):
+ return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % (
+ self.type, self.format, self.repository, self.id,
+ self.revprop['name'])
HEARTBEAT_TIME = 15
class Client(object):
- def __init__(self, pubsub, r, type, repository):
+ def __init__(self, pubsub, r, kind, type, repository):
self.pubsub = pubsub
r.notifyFinish().addErrback(self.finished)
self.r = r
+ self.kind = kind
self.type = type
self.repository = repository
self.alive = True
@@ -123,11 +151,14 @@ class Client(object):
except ValueError:
pass
- def interested_in(self, commit):
- if self.type and self.type != commit.type:
+ def interested_in(self, notification):
+ if self.kind != notification.KIND:
+ return False
+
+ if self.type and self.type != notification.type:
return False
- if self.repository and self.repository != commit.repository:
+ if self.repository and self.repository != notification.repository:
return False
return True
@@ -164,6 +195,13 @@ class SvnPubSub(resource.Resource):
isLeaf = True
clients = []
+ __notification_uri_map = {'commits': Commit.KIND,
+ 'metadata': Metadata.KIND}
+
+ def __init__(self, notification_class):
+ resource.Resource.__init__(self)
+ self.__notification_class = notification_class
+
def cc(self):
return len(self.clients)
@@ -183,6 +221,11 @@ class SvnPubSub(resource.Resource):
request.setResponseCode(400)
return "Invalid path\n"
+ kind = self.__notification_uri_map.get(uri[1], None)
+ if kind is None:
+ request.setResponseCode(400)
+ return "Invalid path\n"
+
if uri_len >= 3:
type = uri[2]
@@ -195,17 +238,18 @@ class SvnPubSub(resource.Resource):
if repository == '*':
repository = None
- c = Client(self, request, type, repository)
+ c = Client(self, request, kind, type, repository)
self.clients.append(c)
c.start()
return twisted.web.server.NOT_DONE_YET
- def notifyAll(self, commit):
- data = commit.render_commit()
+ def notifyAll(self, notification):
+ data = notification.render()
- log.msg("COMMIT: %s (%d clients)" % (commit.render_log(), self.cc()))
+ log.msg("%s: %s (%d clients)"
+ % (notification.KIND, notification.render_log(), self.cc()))
for client in self.clients:
- if client.interested_in(commit):
+ if client.interested_in(notification):
client.write_data(data)
def render_PUT(self, request):
@@ -218,19 +262,23 @@ class SvnPubSub(resource.Resource):
#import pdb;pdb.set_trace()
#print "input: %s" % (input)
try:
- c = json.loads(input)
- commit = Commit(c)
+ data = json.loads(input)
+ notification = self.__notification_class(data)
except ValueError as e:
request.setResponseCode(400)
- log.msg("COMMIT: failed due to: %s" % str(e))
- return str(e)
- self.notifyAll(commit)
+ errstr = str(e)
+ log.msg("%s: failed due to: %s" % (notification.KIND, errstr))
+ return errstr
+ self.notifyAll(notification)
return "Ok"
+
def svnpubsub_server():
root = resource.Resource()
- s = SvnPubSub()
- root.putChild("commits", s)
+ c = SvnPubSub(Commit)
+ m = SvnPubSub(Metadata)
+ root.putChild('commits', c)
+ root.putChild('metadata', m)
return server.Site(root)
if __name__ == "__main__":
diff --git a/tools/server-side/svnpubsub/svnpubsub/util.py b/tools/server-side/svnpubsub/svnpubsub/util.py
new file mode 100644
index 0000000..e254f8b
--- /dev/null
+++ b/tools/server-side/svnpubsub/svnpubsub/util.py
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import subprocess as __subprocess
+
+# check_output() is only available in Python 2.7. Allow us to run with
+# earlier versions
+try:
+ __check_output = __subprocess.check_output
+ def check_output(args, env=None, universal_newlines=False):
+ return __check_output(args, shell=False, env=env,
+ universal_newlines=universal_newlines)
+except AttributeError:
+ def check_output(args, env=None, universal_newlines=False):
+ # note: we only use these three args
+ pipe = __subprocess.Popen(args, shell=False, env=env,
+ stdout=__subprocess.PIPE,
+ universal_newlines=universal_newlines)
+ output, _ = pipe.communicate()
+ if pipe.returncode:
+ raise subprocess.CalledProcessError(pipe.returncode, args)
+ return output
diff --git a/tools/server-side/svnpubsub/svnwcsub.py b/tools/server-side/svnpubsub/svnwcsub.py
index 366df7c..8105d87 100755
--- a/tools/server-side/svnpubsub/svnwcsub.py
+++ b/tools/server-side/svnpubsub/svnwcsub.py
@@ -69,27 +69,22 @@ except ImportError:
import daemonize
import svnpubsub.client
-
-# check_output() is only available in Python 2.7. Allow us to run with
-# earlier versions
-try:
- check_output = subprocess.check_output
-except AttributeError:
- def check_output(args, env): # note: we only use these two args
- pipe = subprocess.Popen(args, stdout=subprocess.PIPE, env=env)
- output, _ = pipe.communicate()
- if pipe.returncode:
- raise subprocess.CalledProcessError(pipe.returncode, args)
- return output
+import svnpubsub.util
assert hasattr(subprocess, 'check_call')
def check_call(*args, **kwds):
- """Wrapper around subprocess.check_call() that logs stderr upon failure."""
+ """Wrapper around subprocess.check_call() that logs stderr upon failure,
+ with an optional list of exit codes to consider non-failure."""
assert 'stderr' not in kwds
+ if '__okayexits' in kwds:
+ __okayexits = kwds['__okayexits']
+ del kwds['__okayexits']
+ else:
+ __okayexits = set([0]) # EXIT_SUCCESS
kwds.update(stderr=subprocess.PIPE)
pipe = subprocess.Popen(*args, **kwds)
output, errput = pipe.communicate()
- if pipe.returncode:
+ if pipe.returncode not in __okayexits:
cmd = args[0] if len(args) else kwds.get('args', '(no command)')
# TODO: log stdout too?
logging.error('Command failed: returncode=%d command=%r stderr=%r',
@@ -103,7 +98,7 @@ def check_call(*args, **kwds):
def svn_info(svnbin, env, path):
"Run 'svn info' on the target path, returning a dict of info data."
args = [svnbin, "info", "--non-interactive", "--", path]
- output = check_output(args, env=env).strip()
+ output = svnpubsub.util.check_output(args, env=env).strip()
info = { }
for line in output.split('\n'):
idx = line.index(':')
@@ -303,6 +298,21 @@ class BackgroundWorker(threading.Thread):
logging.info("updating: %s", wc.path)
+ ## Run the hook
+ HEAD = svn_info(self.svnbin, self.env, wc.url)['Revision']
+ if self.hook:
+ hook_mode = ['pre-update', 'pre-boot'][boot]
+ logging.info('running hook: %s at %s',
+ wc.path, hook_mode)
+ args = [self.hook, hook_mode, wc.path, HEAD, wc.url]
+ rc = check_call(args, env=self.env, __okayexits=[0, 1])
+ if rc == 1:
+ # TODO: log stderr
+ logging.warn('hook denied update of %s at %s',
+ wc.path, hook_mode)
+ return
+ del rc
+
### we need to move some of these args into the config. these are
### still specific to the ASF setup.
args = [self.svnbin, 'switch',
@@ -313,12 +323,13 @@ class BackgroundWorker(threading.Thread):
'--config-option',
'config:miscellany:use-commit-times=on',
'--',
- wc.url,
+ wc.url + '@' + HEAD,
wc.path]
check_call(args, env=self.env)
### check the loglevel before running 'svn info'?
info = svn_info(self.svnbin, self.env, wc.path)
+ assert info['Revision'] == HEAD
logging.info("updated: %s now at r%s", wc.path, info['Revision'])
## Run the hook
@@ -533,7 +544,8 @@ def main(args):
# We manage the logfile ourselves (along with possible rotation). The
# daemon process can just drop stdout/stderr into /dev/null.
- d = Daemon('/dev/null', options.pidfile, options.umask, bdec)
+ d = Daemon('/dev/null', os.path.abspath(options.pidfile),
+ options.umask, bdec)
if options.daemon:
# Daemonize the process and call sys.exit() with appropriate code
d.daemonize_exit()
diff --git a/tools/server-side/svnpubsub/watcher.py b/tools/server-side/svnpubsub/watcher.py
index 340b100..11bf066 100755
--- a/tools/server-side/svnpubsub/watcher.py
+++ b/tools/server-side/svnpubsub/watcher.py
@@ -35,6 +35,9 @@ def _commit(url, commit):
print('COMMIT: from %s' % url)
pprint.pprint(vars(commit), indent=2)
+def _metadata(url, metadata):
+ print('METADATA: from %s' % url)
+ pprint.pprint(vars(metadata), indent=2)
def _event(url, event_name, event_arg):
if event_arg:
@@ -44,7 +47,7 @@ def _event(url, event_name, event_arg):
def main(urls):
- mc = svnpubsub.client.MultiClient(urls, _commit, _event)
+ mc = svnpubsub.client.MultiClient(urls, _commit, _event, _metadata)
mc.run_forever()