Merge with 67e8f639f13c44c1b030f517c97ac08b746b72ea

2 files changed, 435 insertions, 0 deletions

diff --git a/extras/buildbot/googlecode_atom.py b/extras/buildbot/googlecode_atom.py
new file mode 100644
index 0000000..0d4631f
--- /dev/null
+++ b/extras/buildbot/googlecode_atom.py
@@ -0,0 +1,171 @@
+# GoogleCode Atom Feed Poller
+# Author: Srivats P. <pstavirs>
+# Based on Mozilla's HgPoller
+#     http://bonsai.mozilla.org/cvsblame.cgi?file=/mozilla/tools/buildbot/buildbot/changes/Attic/hgpoller.py&revision=1.1.4.2
+#
+# Description:
+#   Use this ChangeSource for projects hosted on http://code.google.com/
+#
+#   This ChangeSource uses the project's commit Atom feed. Depending upon the
+#   frequency of commits, you can tune the polling interval for the feed
+#   (default is 1 hour)
+#
+# Parameters:
+#   feedurl (MANDATORY): The Atom feed URL of the GoogleCode repo
+#   pollinterval (OPTIONAL): Polling frequency for the feed (in seconds)
+#
+# Example:
+# To poll the Ostinato project's commit feed every 3 hours, use -
+#   from googlecode_atom import GoogleCodeAtomPoller
+#   poller = GoogleCodeAtomPoller(
+#            feedurl="http://code.google.com/feeds/p/ostinato/hgchanges/basic",
+#            pollinterval=10800)
+#   c['change_source'] = [ poller ]
+#
+
+from time import strptime
+from calendar import timegm
+from xml.dom import minidom, Node
+
+from twisted.python import log, failure
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+from twisted.web.client import getPage
+
+from buildbot.changes import base, changes
+
+def googleCodePollerForProject(project, vcs, pollinterval=3600):
+    return GoogleCodeAtomPoller(
+        'http://code.google.com/feeds/p/%s/%schanges/basic' % (project, vcs),
+        pollinterval=pollinterval)
+
+
+class GoogleCodeAtomPoller(base.ChangeSource):
+    """This source will poll a GoogleCode Atom feed for changes and
+    submit them to the change master. Works for both Svn and Hg repos.
+    TODO: branch processing
+    """
+
+    compare_attrs = ['feedurl', 'pollinterval']
+    parent = None
+    loop = None
+    volatile = ['loop']
+    working = False
+
+    def __init__(self, feedurl, pollinterval=3600):
+        """
+        @type   feedurl:        string
+        @param  feedurl:        The Atom feed URL of the GoogleCode repo
+                                (e.g. http://code.google.com/feeds/p/ostinato/hgchanges/basic)
+
+        @type   pollinterval:   int
+        @param  pollinterval:   The time (in seconds) between queries for
+                                changes (default is 1 hour)
+        """
+
+        self.feedurl = feedurl
+        self.branch = None
+        self.pollinterval = pollinterval
+        self.lastChange = None
+        self.loop = LoopingCall(self.poll)
+
+    def startService(self):
+        log.msg("GoogleCodeAtomPoller starting")
+        base.ChangeSource.startService(self)
+        reactor.callLater(0, self.loop.start, self.pollinterval)
+
+    def stopService(self):
+        log.msg("GoogleCodeAtomPoller stoppping")
+        self.loop.stop()
+        return base.ChangeSource.stopService(self)
+
+    def describe(self):
+        return ("Getting changes from the GoogleCode repo changes feed %s" %
+               self._make_url())
+
+    def poll(self):
+        if self.working:
+            log.msg("Not polling because last poll is still working")
+        else:
+            self.working = True
+            d = self._get_changes()
+            d.addCallback(self._process_changes)
+            d.addCallbacks(self._finished_ok, self._finished_failure)
+
+    def _finished_ok(self, res):
+        assert self.working
+        self.working = False
+        log.msg("GoogleCodeAtomPoller poll success")
+
+        return res
+
+    def _finished_failure(self, res):
+        log.msg("GoogleCodeAtomPoller poll failed: %s" % res)
+        assert self.working
+        self.working = False
+        return None
+
+    def _make_url(self):
+        return "%s" % (self.feedurl)
+
+    def _get_changes(self):
+        url = self._make_url()
+        log.msg("GoogleCodeAtomPoller polling %s" % url)
+
+        return getPage(url, timeout=self.pollinterval)
+
+    def _parse_changes(self, query):
+        dom = minidom.parseString(query)
+        entries = dom.getElementsByTagName("entry")
+        changes = []
+        # Entries come in reverse chronological order
+        for i in entries:
+            d = {}
+
+            # revision is the last part of the 'id' url
+            d["revision"] = i.getElementsByTagName(
+                "id")[0].firstChild.data.split('/')[-1]
+            if d["revision"] == self.lastChange:
+                break  # no more new changes
+
+            d["when"] = timegm(strptime(
+                i.getElementsByTagName("updated")[0].firstChild.data,
+                "%Y-%m-%dT%H:%M:%SZ"))
+            d["author"] = i.getElementsByTagName(
+                "author")[0].getElementsByTagName("name")[0].firstChild.data
+            # files and commit msg are separated by 2 consecutive <br/>
+            content = i.getElementsByTagName(
+                "content")[0].firstChild.data.split("<br/>\n <br/>")
+            # Remove the action keywords from the file list
+            fl = content[0].replace(
+                u' \xa0\xa0\xa0\xa0Add\xa0\xa0\xa0\xa0', '').replace(
+                u' \xa0\xa0\xa0\xa0Delete\xa0\xa0\xa0\xa0', '').replace(
+                u' \xa0\xa0\xa0\xa0Modify\xa0\xa0\xa0\xa0', '')
+            # Get individual files and remove the 'header'
+            d["files"] = fl.encode("ascii", "replace").split("<br/>")[1:]
+            d["files"] = [f.strip() for f in d["files"]]
+            try:
+                d["comments"] = content[1].encode("ascii", "replace")
+            except:
+                d["comments"] = "No commit message provided"
+
+            changes.append(d)
+
+        changes.reverse() # want them in chronological order
+        return changes
+
+    def _process_changes(self, query):
+        change_list = self._parse_changes(query)
+
+        # Skip calling addChange() if this is the first successful poll.
+        if self.lastChange is not None:
+            for change in change_list:
+                c = changes.Change(revision = change["revision"],
+                                   who = change["author"],
+                                   files = change["files"],
+                                   comments = change["comments"],
+                                   when = change["when"],
+                                   branch = self.branch)
+                self.parent.addChange(c)
+        if change_list:
+            self.lastChange = change_list[-1]["revision"]
diff --git a/extras/buildbot/master.cfg b/extras/buildbot/master.cfg
new file mode 100644
index 0000000..92403fe
--- /dev/null
+++ b/extras/buildbot/master.cfg
@@ -0,0 +1,264 @@
+# -*- python -*-
+# ex: set syntax=python:
+
+# This is a sample buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory (although the filename
+# can be changed with the --basedir option to 'mktap buildbot master').
+
+# It has one job: define a dictionary named BuildmasterConfig. This
+# dictionary has a variety of keys to control different aspects of the
+# buildmaster. They are documented in docs/config.xhtml .
+
+PYTHON_VERSIONS = ('2.4', '2.5', '2.6', '2.7', '3.2')
+
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+####### DB URL
+
+# This specifies what database buildbot uses to store change and scheduler
+# state
+c['db_url'] = "sqlite:///state.sqlite"
+
+####### BUILDSLAVES
+
+# the 'slaves' list defines the set of allowable buildslaves. Each element is
+# a BuildSlave object, which is created with bot-name, bot-password.  These
+# correspond to values given to the buildslave's mktap invocation.
+from buildbot.buildslave import BuildSlave
+c['slaves'] = [BuildSlave("bot1linux", "imtheslave")]
+
+# to limit to two concurrent builds on a slave, use
+#  c['slaves'] = [BuildSlave("bot1name", "bot1passwd", max_builds=2)]
+
+
+# 'slavePortnum' defines the TCP port to listen on. This must match the value
+# configured into the buildslaves (with their --master option)
+
+c['slavePortnum'] = 9989
+
+####### CHANGESOURCES
+
+# the 'change_source' setting tells the buildmaster how it should find out
+# about source code changes. Any class which implements IChangeSource can be
+# put here: there are several in buildbot/changes/*.py to choose from.
+
+from buildbot.changes.pb import PBChangeSource
+c['change_source'] = PBChangeSource()
+
+from googlecode_atom import GoogleCodeAtomPoller
+poller = GoogleCodeAtomPoller(
+    feedurl="http://code.google.com/feeds/p/python-sqlparse/hgchanges/basic",
+    pollinterval=600)
+c['change_source'] = [ poller ]
+
+# For example, if you had CVSToys installed on your repository, and your
+# CVSROOT/freshcfg file had an entry like this:
+#pb = ConfigurationSet([
+#    (None, None, None, PBService(userpass=('foo', 'bar'), port=4519)),
+#    ])
+
+# then you could use the following buildmaster Change Source to subscribe to
+# the FreshCVS daemon and be notified on every commit:
+#
+#from buildbot.changes.freshcvs import FreshCVSSource
+#fc_source = FreshCVSSource("cvs.example.com", 4519, "foo", "bar")
+#c['change_source'] = fc_source
+
+# or, use a PBChangeSource, and then have your repository's commit script run
+# 'buildbot sendchange', or use contrib/svn_buildbot.py, or
+# contrib/arch_buildbot.py :
+#
+#from buildbot.changes.pb import PBChangeSource
+#c['change_source'] = PBChangeSource()
+
+# If you wat to use SVNPoller, it might look something like
+#  # Where to get source code changes
+# from buildbot.changes.svnpoller import SVNPoller
+# source_code_svn_url='https://svn.myproject.org/bluejay/trunk'
+# svn_poller = SVNPoller(
+#                    svnurl=source_code_svn_url,
+#                    pollinterval=60*60, # seconds
+#                    histmax=10,
+#                    svnbin='/usr/bin/svn',
+## )
+# c['change_source'] = [ svn_poller ]
+
+####### SCHEDULERS
+
+## configure the Schedulers
+
+from buildbot.scheduler import Scheduler
+c['schedulers'] = []
+for py_ver in PYTHON_VERSIONS:
+    c['schedulers'].append(
+        Scheduler(name="py%s" % py_ver, branch=None,
+                  treeStableTimer=2*60,
+                  builderNames=["builder-%s" % py_ver]))
+
+
+####### BUILDERS
+
+# the 'builders' list defines the Builders. Each one is configured with a
+# dictionary, using the following keys:
+#  name (required): the name used to describe this builder
+#  slavename or slavenames (required): which slave(s) to use (must appear in c['slaves'])
+#  factory (required): a BuildFactory to define how the build is run
+#  builddir (optional): which subdirectory to run the builder in
+
+# buildbot/process/factory.py provides several BuildFactory classes you can
+# start with, which implement build processes for common targets (GNU
+# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
+# base class, and is configured with a series of BuildSteps. When the build
+# is run, the appropriate buildslave is told to execute each Step in turn.
+
+# the first BuildStep is typically responsible for obtaining a copy of the
+# sources. There are source-obtaining Steps in buildbot/steps/source.py for
+# CVS, SVN, and others.
+
+cvsroot = ":pserver:anonymous@cvs.sourceforge.net:/cvsroot/buildbot"
+cvsmodule = "buildbot"
+
+from buildbot.process import factory
+from buildbot.steps.source import CVS, Mercurial
+from buildbot.steps.shell import Compile, ShellCommand
+from buildbot.steps.python_twisted import Trial
+f1 = factory.BuildFactory()
+f1.addStep(CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, login="", mode="copy"))
+f1.addStep(Compile(command=["python", "./setup.py", "build"]))
+f1.addStep(Trial(testChanges=True, testpath="."))
+
+from buildbot.config import BuilderConfig
+
+def _mk_factory(py_ver):
+    py_bin = "/home/build/python/python%(ver)s/bin/python%(ver)s" % {"ver": py_ver}
+    py2to3 = "/home/build/python/python%(ver)s/bin/2to3" % {"ver": py_ver}
+    site_pkgs = "/home/build/python/python%(ver)s/lib/site-packages/sqlparse" % {"ver": py_ver}
+    is_py3k = bool(py_ver.startswith("3"))
+    workdir = "build/"
+    f = factory.BuildFactory()
+    f.addStep(Mercurial(repourl="http://python-sqlparse.googlecode.com/hg/"))
+    f.addStep(ShellCommand(command=["rm", "-rf", site_pkgs],
+                           description="removing installed package",
+                           descriptionDone="site-pkgs clean"))
+    if is_py3k:
+        workdir = "build/extras/py3k/"
+        f.addStep(ShellCommand(command=["make", "clean"],
+                               workdir=workdir,
+                               description="cleaning up",
+                               descriptionDone="cleaned up"))
+        f.addStep(ShellCommand(command=["make", "2TO3=%s" % py2to3],
+                               workdir=workdir,
+                               description="creating py3 version",
+                               descriptionDone="py3 version created"))
+    f.addStep(Compile(command=[py_bin, "setup.py", "build"],
+                      workdir=workdir))
+    f.addStep(ShellCommand(command=[py_bin, "setup.py", "install"],
+                           description="installing module",
+                           descriptionDone="module installed",
+                           workdir=workdir))
+    f.addStep(ShellCommand(command=["mv", "sqlparse", "_sqlparse"],
+                           description="moving local module",
+                           descriptionDone="local module moved",
+                           workdir=workdir))
+    f.addStep(ShellCommand(command=[py_bin, "tests/run_tests.py"],
+                           description="running tests",
+                           descriptionDone="tests done",
+                           workdir=workdir))
+    f.addStep(ShellCommand(command=["mv", "_sqlparse", "sqlparse"],
+                           description="restoring local module",
+                           descriptionDone="local module restored",
+                           workdir=workdir))
+    return f
+
+def _mk_builder(py_ver):
+    return BuilderConfig(
+        name="builder-%s" % py_ver,
+        slavename="bot1linux",
+        builddir="full-%s" % py_ver,
+        factory=_mk_factory(py_ver))
+
+c['builders'] = []
+for py_ver in PYTHON_VERSIONS:
+    c['builders'].append(_mk_builder(py_ver))
+
+
+####### STATUS TARGETS
+
+# 'status' is a list of Status Targets. The results of each build will be
+# pushed to these targets. buildbot/status/*.py has a variety to choose from,
+# including web pages, email senders, and IRC bots.
+
+c['status'] = []
+
+from buildbot.status import html
+from buildbot.status.web import auth, authz
+authz_cfg=authz.Authz(
+    # change any of these to True to enable; see the manual for more
+    # options
+    gracefulShutdown = False,
+    forceBuild = True,
+    forceAllBuilds = True,
+    pingBuilder = True,
+    stopBuild = False,
+    stopAllBuilds = False,
+    cancelPendingBuild = True,
+)
+c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+
+from buildbot.status import mail
+c['status'].append(mail.MailNotifier(
+    fromaddr="buildbot@andialbrecht.de",
+    extraRecipients=["albrecht.andi@gmail.com"],
+    sendToInterestedUsers=False,
+    mode="failing"))
+#
+# from buildbot.status import words
+# c['status'].append(words.IRC(host="irc.example.com", nick="bb",
+#                              channels=["#example"]))
+# c['status'].append(words.IRC(host="irc.example.com", nick="bb",
+#                              channels=["#example"], useSSL=True))
+#
+# from buildbot.status import client
+# c['status'].append(client.PBListener(9988))
+
+
+####### DEBUGGING OPTIONS
+
+# if you set 'debugPassword', then you can connect to the buildmaster with
+# the diagnostic tool in contrib/debugclient.py . From this tool, you can
+# manually force builds and inject changes, which may be useful for testing
+# your buildmaster without actually committing changes to your repository (or
+# before you have a functioning 'sources' set up). The debug tool uses the
+# same port number as the slaves do: 'slavePortnum'.
+
+#c['debugPassword'] = "debugpassword"
+
+# if you set 'manhole', you can ssh into the buildmaster and get an
+# interactive python shell, which may be useful for debugging buildbot
+# internals. It is probably only useful for buildbot developers. You can also
+# use an authorized_keys file, or plain telnet.
+#from buildbot import manhole
+#c['manhole'] = manhole.PasswordManhole("tcp:9999:interface=127.0.0.1",
+#                                       "admin", "password")
+
+
+####### PROJECT IDENTITY
+
+# the 'projectName' string will be used to describe the project that this
+# buildbot is working on. For example, it is used as the title of the
+# waterfall HTML page. The 'projectURL' string will be used to provide a link
+# from buildbot HTML pages to your project's home page.
+
+c['projectName'] = "python-sqlparse"
+c['projectURL'] = "http://python-sqlparse.googlecode.com"
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.WebStatus page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = "http://buildbot.andialbrecht.de"