diff options
Diffstat (limited to 'Tools/EWSTools')
18 files changed, 648 insertions, 96 deletions
diff --git a/Tools/EWSTools/GoogleComputeEngine/build-chromium-ews.sh b/Tools/EWSTools/GoogleComputeEngine/build-chromium-ews.sh index eb9355b53..6c9e7450f 100755 --- a/Tools/EWSTools/GoogleComputeEngine/build-chromium-ews.sh +++ b/Tools/EWSTools/GoogleComputeEngine/build-chromium-ews.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012 Google Inc. All rights reserved. +# Copyright (c) 2013 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -11,9 +11,6 @@ # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -32,18 +29,32 @@ echo "Usage: build-cr-linux-ews.sh BOT_NUMBER" exit 1 fi +CWD="$(pwd)" +cd "$(dirname "$0")" + QUEUE_TYPE=chromium-ews BOT_ID=gce-cr-linux-$1 BUGZILLA_USERNAME=webkit.review.bot@gmail.com read -s -p "Bugzilla Password: " BUGZILLA_PASSWORD && echo -# FIXME: We should use gcutil to find a zone that's actually up. -ZONE=us-east-b -IMAGE=projects/google/images/ubuntu-10-04-v20120621 +PROJECT=google.com:webkit +ZONE=$(bash findzone.sh $PROJECT) +IMAGE=projects/google/global/images/gcel-10-04-v20130104 +MACHINE_TYPE=n1-standard-4-d -gcutil addinstance $BOT_ID --machine_type=standard-4-cpu-ephemeral-disk --image=$IMAGE --zone=$ZONE --wait_until_running +gcutil --project=$PROJECT addinstance $BOT_ID --machine_type=$MACHINE_TYPE --image=$IMAGE --zone=$ZONE --wait_until_running echo "Sleeping for 30s to let the server spin up ssh..." sleep 30 -gcutil ssh $BOT_ID "sudo apt-get install subversion -y && svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools && cd tools && bash cold-boot.sh $QUEUE_TYPE $BOT_ID $BUGZILLA_USERNAME $BUGZILLA_PASSWORD" +gcutil --project=$PROJECT ssh $BOT_ID " + sudo apt-get install subversion -y && + svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools && + cd tools && + bash build-vm.sh && + bash build-repo.sh $QUEUE_TYPE $BUGZILLA_USERNAME $BUGZILLA_PASSWORD && + bash build-boot-cmd.sh \"screen -t kr ./start-queue.sh $QUEUE_TYPE $BOT_ID 10\" && + bash boot.sh +" + +cd "$CWD" diff --git a/Tools/EWSTools/GoogleComputeEngine/build-commit-queue.sh b/Tools/EWSTools/GoogleComputeEngine/build-commit-queue.sh index a94b02083..4ac73c8be 100755 --- a/Tools/EWSTools/GoogleComputeEngine/build-commit-queue.sh +++ b/Tools/EWSTools/GoogleComputeEngine/build-commit-queue.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012 Google Inc. All rights reserved. +# Copyright (c) 2013 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -11,9 +11,6 @@ # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -32,21 +29,36 @@ echo "Usage: build-commit-queue.sh BOT_NUMBER" exit 1 fi +CWD="$(pwd)" +cd "$(dirname "$0")" + QUEUE_TYPE=commit-queue BOT_ID=gce-cq-$1 BUGZILLA_USERNAME=webkit.review.bot@gmail.com read -s -p "Bugzilla Password: " BUGZILLA_PASSWORD && echo - SVN_USERNAME=commit-queue@webkit.org read -s -p "Subversion Password: " SVN_PASSWORD && echo -# FIXME: We should use gcutil to find a zone that's actually up. -ZONE=us-east-b -IMAGE=projects/google/images/ubuntu-10-04-v20120621 +PROJECT=google.com:webkit +ZONE=$(bash findzone.sh $PROJECT) +IMAGE=projects/google/global/images/gcel-10-04-v20130104 +MACHINE_TYPE=n1-standard-4-d -gcutil addinstance $BOT_ID --machine_type=standard-4-cpu-ephemeral-disk --image=$IMAGE --zone=$ZONE --wait_until_running +gcutil --project=$PROJECT addinstance $BOT_ID --machine_type=$MACHINE_TYPE --image=$IMAGE --zone=$ZONE --wait_until_running echo "Sleeping for 30s to let the server spin up ssh..." sleep 30 -gcutil ssh $BOT_ID "sudo apt-get install subversion -y && svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools && cd tools && bash configure-svn-auth.sh $SVN_USERNAME $SVN_PASSWORD && bash cold-boot.sh $QUEUE_TYPE $BOT_ID $BUGZILLA_USERNAME $BUGZILLA_PASSWORD" +gcutil --project=$PROJECT ssh $BOT_ID " + sudo apt-get install subversion -y && + svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools && + cd tools && + bash configure-svn-config.sh && + bash configure-svn-auth.sh $SVN_USERNAME $SVN_PASSWORD && + bash build-vm.sh && + bash build-repo.sh $QUEUE_TYPE $BUGZILLA_USERNAME $BUGZILLA_PASSWORD && + bash build-boot-cmd.sh \"screen -t kr ./start-queue.sh $QUEUE_TYPE $BOT_ID 10\" && + bash boot.sh +" + +cd "$CWD" diff --git a/Tools/EWSTools/GoogleComputeEngine/build-cr-linux-debug-ews.sh b/Tools/EWSTools/GoogleComputeEngine/build-cr-linux-debug-ews.sh new file mode 100755 index 000000000..1cf4e6902 --- /dev/null +++ b/Tools/EWSTools/GoogleComputeEngine/build-cr-linux-debug-ews.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -ne 1 ]];then +echo "Usage: build-cr-linux-debug-ews.sh BOT_NUMBER" +exit 1 +fi + +CWD="$(pwd)" +cd "$(dirname "$0")" + +QUEUE_TYPE=cr-linux-debug-ews +BOT_ID=gce-cr-linux-debug-$1 +BUGZILLA_USERNAME=webkit.review.bot@gmail.com +read -s -p "Bugzilla Password: " BUGZILLA_PASSWORD && echo + +PROJECT=google.com:webkit +ZONE=$(bash findzone.sh $PROJECT) +IMAGE=projects/google/global/images/gcel-10-04-v20130104 +MACHINE_TYPE=n1-standard-4-d + +gcutil --project=$PROJECT addinstance $BOT_ID --machine_type=$MACHINE_TYPE --image=$IMAGE --zone=$ZONE --wait_until_running + +echo "Sleeping for 30s to let the server spin up ssh..." +sleep 30 + +gcutil --project=$PROJECT ssh $BOT_ID " + sudo apt-get install subversion -y && + svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools && + cd tools && + bash build-vm.sh && + bash build-repo.sh $QUEUE_TYPE $BUGZILLA_USERNAME $BUGZILLA_PASSWORD && + bash build-boot-cmd.sh \"\\ +screen -t kr ./start-queue.sh -r \\\"configure-clang-linux.sh $QUEUE_TYPE\\\" $QUEUE_TYPE $BOT_ID 10\" && + bash boot.sh +" + +cd "$CWD" diff --git a/Tools/EWSTools/GoogleComputeEngine/build-feeder-style-sheriffbot.sh b/Tools/EWSTools/GoogleComputeEngine/build-feeder-style-sheriffbot.sh new file mode 100755 index 000000000..6297d80cf --- /dev/null +++ b/Tools/EWSTools/GoogleComputeEngine/build-feeder-style-sheriffbot.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -ne 1 ]];then +echo "Usage: build-feeder-style-sheriffbot.sh BOT_NUMBER" +exit 1 +fi + +CWD="$(pwd)" +cd "$(dirname "$0")" + +BOT_ID=gce-feeder-$1 +BUGZILLA_USERNAME=webkit.review.bot@gmail.com +read -s -p "Bugzilla Password: " BUGZILLA_PASSWORD && echo +read -s -p "sheriffbot IRC Password: " IRC_PASSWORD && echo + +PROJECT=google.com:webkit +ZONE=$(bash findzone.sh $PROJECT) +IMAGE=projects/google/global/images/gcel-10-04-v20130104 +MACHINE_TYPE=n1-standard-4-d + +gcutil --project=$PROJECT addinstance $BOT_ID --machine_type=$MACHINE_TYPE --image=$IMAGE --zone=$ZONE --wait_until_running + +echo "Sleeping for 30s to let the server spin up ssh..." +sleep 30 + +gcutil --project=$PROJECT ssh $BOT_ID " + sudo apt-get install subversion -y && + svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools && + cd tools && + bash build-vm.sh && + bash build-repo.sh feeder-queue $BUGZILLA_USERNAME $BUGZILLA_PASSWORD && + cp -r /mnt/git/webkit-feeder-queue /mnt/git/webkit-style-queue && + cp -r /mnt/git/webkit-feeder-queue /mnt/git/webkit-sheriff-bot && + bash configure-svn-config.sh && + bash configure-git-svn.sh sheriff-bot && + bash configure-git-user.sh sheriff-bot \"Sheriff Bot\" $BUGZILLA_USERNAME && + bash build-boot-cmd.sh \"\\ +screen -t fq ./start-queue.sh feeder-queue $BOT_ID 10 +screen -t sq ./start-queue.sh style-queue $BOT_ID 10 +screen -t sb ./start-queue.sh -p \\\"--irc-password=$IRC_PASSWORD\\\" sheriff-bot $BOT_ID 180\" && + bash boot.sh +" + +cd "$CWD" diff --git a/Tools/EWSTools/GoogleComputeEngine/findzone.sh b/Tools/EWSTools/GoogleComputeEngine/findzone.sh new file mode 100755 index 000000000..ac2cee5fa --- /dev/null +++ b/Tools/EWSTools/GoogleComputeEngine/findzone.sh @@ -0,0 +1,5 @@ +if [[ $# -ne 1 ]]; then + echo "Usage: findzone.sh PROJECT" +fi + +echo $(gcutil --project=$1 listzones | grep UP | awk '{print $2}' | sort | tail -1) diff --git a/Tools/EWSTools/boot.sh b/Tools/EWSTools/boot.sh index fb5aadde0..787e61994 100755 --- a/Tools/EWSTools/boot.sh +++ b/Tools/EWSTools/boot.sh @@ -27,4 +27,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -screen -d -m -c ~/tools/screen-config +CWD=$(pwd) + +cd ~/tools +screen -d -m -c ./screen-config + +cd $CWD diff --git a/Tools/EWSTools/create-webkit-git b/Tools/EWSTools/build-boot-cmd.sh index cbf8ce4f2..b22c1fbb7 100755 --- a/Tools/EWSTools/create-webkit-git +++ b/Tools/EWSTools/build-boot-cmd.sh @@ -1,5 +1,5 @@ -#/bin/bash -# Copyright (c) 2010 Google Inc. All rights reserved. +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,16 +27,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# These are meant to match the instructions from: -# http://trac.webkit.org/wiki/UsingGitWithWebKit -cd /mnt/git -git clone git://git.webkit.org/WebKit.git webkit -cd webkit +if [[ $# -ne 1 ]];then +echo "Usage: build-boot-cmd.sh BOOT_COMMAND" +exit 1 +fi -git svn init -T trunk http://svn.webkit.org/repository/webkit -git update-ref refs/remotes/trunk origin/master -# It's possible that this "config" step can get merged into an earlier setup step. -git config --replace-all svn-remote.svn.fetch trunk:refs/remotes/origin/master - -git fetch -git svn rebase +echo "$1" > ~/tools/screen-config diff --git a/Tools/EWSTools/build-repo.sh b/Tools/EWSTools/build-repo.sh new file mode 100755 index 000000000..65b0d84e9 --- /dev/null +++ b/Tools/EWSTools/build-repo.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -ne 3 ]];then +echo "Usage: build-repo.sh QUEUE_TYPE BUGZILLA_USERNAME BUGZILLA_PASSWORD" +exit 1 +fi + +CWD=$(pwd) + +cd /mnt/git + +echo "Cloning WebKit git repository, process takes ~30m." +echo "Note: No status output will be shown via remote pipe." +git clone git://git.webkit.org/WebKit.git webkit-$1 +cd webkit-$1 + +cat >> .git/config <<EOF +[bugzilla] + username = $2 + password = $3 +EOF + +if [[ $1 == "commit-queue" ]];then +cat >> .git/config <<EOF +[svn-remote "svn"] + url = http://svn.webkit.org/repository/webkit + fetch = trunk:refs/remotes/origin/master +[user] + email = commit-queue@webkit.org + name = Commit Queue +EOF +fi + +cd $CWD diff --git a/Tools/EWSTools/cold-boot.sh b/Tools/EWSTools/build-vm.sh index 81516b4b0..62daca007 100755 --- a/Tools/EWSTools/cold-boot.sh +++ b/Tools/EWSTools/build-vm.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012 Google Inc. All rights reserved. +# Copyright (c) 2013 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -11,9 +11,6 @@ # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -27,13 +24,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -if [[ $# -ne 4 ]];then -echo "Usage: cold-boot.sh QUEUE_TYPE BOT_ID BUGZILLA_USERNAME BUGZILLA_PASSWORD" +if [[ $# -ne 0 ]];then +echo "Usage: build-vm.sh" exit 1 fi +CWD=$(pwd) + # Format the disk -cat <<EOF | sudo fdisk /dev/vdb +cat <<EOF | sudo fdisk /dev/sdb n p 1 @@ -42,8 +41,8 @@ p w EOF -sudo mkfs.ext4 /dev/vdb1 -sudo mount /dev/vdb1 /mnt +sudo mkfs.ext4 /dev/sdb1 +sudo mount /dev/sdb1 /mnt echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections @@ -58,29 +57,7 @@ cd /mnt sudo mkdir -p git sudo chown $USER git sudo chgrp $USER git -cd git - -git clone http://git.chromium.org/external/Webkit.git -mv Webkit webkit-$1 -cd webkit-$1 - -cat >> .git/config <<EOF -[bugzilla] - username = $3 - password = $4 -EOF -if [[ $1 == "commit-queue" ]];then -cat >> .git/config <<EOF -[svn-remote "svn"] - url = http://svn.webkit.org/repository/webkit - fetch = trunk:refs/remotes/origin/master -[user] - email = commit-queue@webkit.org - name = Commit Queue -EOF -fi +sudo chmod 644 /etc/hosts -cd ~/tools -echo "screen -t kr ./start-queue.sh" $1 $2 > screen-config -bash boot.sh +cd $CWD diff --git a/Tools/EWSTools/configure-clang-linux.sh b/Tools/EWSTools/configure-clang-linux.sh new file mode 100755 index 000000000..049c206eb --- /dev/null +++ b/Tools/EWSTools/configure-clang-linux.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -ne 1 ]];then + echo "Usage: configure-clang-linux.sh QUEUE_TYPE" + exit 1 +fi + +bash /mnt/git/$QUEUE_TYPE/Source/WebKit/chromium/tools/clang/scripts/update.sh +export builddir_name=llvm +export GYP_DEFINES='clang=1'
\ No newline at end of file diff --git a/Tools/EWSTools/configure-git-svn.sh b/Tools/EWSTools/configure-git-svn.sh new file mode 100755 index 000000000..0416125ca --- /dev/null +++ b/Tools/EWSTools/configure-git-svn.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -ne 1 ]]; then + echo "Usage: configure-git-svn.sh QUEUE_TYPE" + exit 1 +fi + +QUEUE_TYPE=$1 + +CWD="$(pwd)" +cd /mnt/git/webkit-$QUEUE_TYPE + +# These commands come from the WebKit wiki: http://trac.webkit.org/wiki/UsingGitWithWebKit +git svn init --prefix=origin/ -T trunk http://svn.webkit.org/repository/webkit +git config --replace svn-remote.svn.fetch trunk:refs/remotes/origin/master +git svn fetch --local + +cd "$CWD"
\ No newline at end of file diff --git a/Tools/EWSTools/configure-git-user.sh b/Tools/EWSTools/configure-git-user.sh new file mode 100755 index 000000000..0c29360aa --- /dev/null +++ b/Tools/EWSTools/configure-git-user.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -ne 3 ]]; then + echo "Usage: configure-git-user.sh QUEUE_TYPE NAME EMAIL" + exit 1 +fi + +QUEUE_TYPE=$1 +NAME="$2" +EMAIL=$3 + +if [[ "$NAME" != *" "* ]]; then + echo "NAME parameter must contain a space." + exit 1 +fi + +cat >> /mnt/git/webkit-$QUEUE_TYPE/.git/config <<EOF +[user] + name = $NAME + email = $EMAIL +EOF diff --git a/Tools/EWSTools/configure-svn-config.sh b/Tools/EWSTools/configure-svn-config.sh new file mode 100755 index 000000000..6054e4732 --- /dev/null +++ b/Tools/EWSTools/configure-svn-config.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +svn export http://svn.webkit.org/repository/webkit/trunk/Tools/svn-config ~/.subversion/config diff --git a/Tools/EWSTools/screen-config b/Tools/EWSTools/screen-config index 5c003df2e..5a00fee96 100644 --- a/Tools/EWSTools/screen-config +++ b/Tools/EWSTools/screen-config @@ -1,4 +1,4 @@ -screen -t style ./start-queue.sh style-queue -screen -t qt ./start-queue.sh qt-ews -screen -t kr ./start-queue.sh chromium-ews -screen -t gtk ./start-queue.sh gtk-ews +screen -t style ./start-queue.sh style-queue 10 +screen -t qt ./start-queue.sh qt-ews 10 +screen -t kr ./start-queue.sh chromium-ews 10 +screen -t gtk ./start-queue.sh gtk-ews 10 diff --git a/Tools/EWSTools/setup-commit-queue.sh b/Tools/EWSTools/setup-commit-queue.sh new file mode 100755 index 000000000..dd17862c9 --- /dev/null +++ b/Tools/EWSTools/setup-commit-queue.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Copyright (c) 2013 Google Inc. All rights reserved. +# Copyright (C) 2013 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +CWD="$(pwd)" +cd "$(dirname "$0")" + +QUEUE_TYPE=commit-queue +BUGZILLA_USERNAME=commit-queue@webkit.org +read -s -p "Bugzilla Password: " BUGZILLA_PASSWORD && echo +SVN_USERNAME=commit-queue@webkit.org +read -s -p "Subversion Password: " SVN_PASSWORD && echo + +svn checkout http://svn.webkit.org/repository/webkit/trunk/Tools/EWSTools tools +bash tools/configure-svn-config.sh +bash tools/configure-svn-auth.sh $SVN_USERNAME $SVN_PASSWORD + +echo "Cloning WebKit git repository, process takes ~30m." +echo "Note: No status output will be shown via remote pipe." +git clone git://git.webkit.org/WebKit.git WebKit +cd WebKit + +cat >> .git/config <<EOF +[bugzilla] + username = $BUGZILLA_USERNAME + password = $BUGZILLA_PASSWORD +[svn-remote "svn"] + url = http://svn.webkit.org/repository/webkit + fetch = trunk:refs/remotes/origin/master +[user] + email = commit-queue@webkit.org + name = Commit Queue +EOF diff --git a/Tools/EWSTools/start-queue-mac.sh b/Tools/EWSTools/start-queue-mac.sh new file mode 100755 index 000000000..5fc05a317 --- /dev/null +++ b/Tools/EWSTools/start-queue-mac.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# Copyright (c) 2012 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [[ $# -lt 3 ]]; then + echo "Usage: start-queue-loop.sh QUEUE_NAME BOT_ID RESET_AFTER_ITERATION [QUEUE_PARAMS]" + exit 1 +fi + +QUEUE_NAME=$1 +BOT_ID=$2 +RESET_AFTER_ITERATION=$3 +shift 3 +QUEUE_PARAMS="$@" + +while : +do + # Delete log files older than 14 days, move aside the main mac-ews.log file to prevent it from growing extra large. + cd /Volumes/Data/EWS/$QUEUE_NAME-logs + find . -mtime +14 -delete + rm $QUEUE_NAME.old + mv $QUEUE_NAME.log $QUEUE_NAME.old + cd /Volumes/Data/EWS/Webkit + + # Delete WebKitBuild to force a clean build + rm -rf /Volumes/Data/EWS/WebKit/WebKitBuild + + # This somewhat quirky sequence of steps seems to clear up all the broken + # git situations we've gotten ourself into in the past. + git clean -f # Remove any left-over layout test results, added files, etc. + git rebase --abort # If we got killed during a git rebase, we need to clean up. + git fetch origin # Avoid updating the working copy to a stale revision. + git checkout origin/master -f + git branch -D master + git checkout origin/master -b master + + # Most queues auto-update as part of their normal operation, but updating + # here makes sure that we get the latest version of the master process. + ./Tools/Scripts/update-webkit + + # test-webkitpy has code to remove orphaned .pyc files, so we + # run it before running webkit-patch to avoid stale .pyc files + # preventing webkit-patch from launching. + ./Tools/Scripts/test-webkitpy + + # We use --exit-after-iteration to pick up any changes to webkit-patch, including + # changes to the contributors.json file. + ./Tools/Scripts/webkit-patch $QUEUE_NAME --bot-id=$BOT_ID --no-confirm --exit-after-iteration $RESET_AFTER_ITERATION $QUEUE_PARAMS +done diff --git a/Tools/EWSTools/start-queue-win.sh b/Tools/EWSTools/start-queue-win.sh new file mode 100755 index 000000000..bd08cd117 --- /dev/null +++ b/Tools/EWSTools/start-queue-win.sh @@ -0,0 +1,52 @@ +#!/bin/sh +export WEBKIT_BUGZILLA_USERNAME=$1 +export WEBKIT_BUGZILLA_PASSWORD=$2 +export BOT_ID=$3 + +function error_handler() +{ + start_ews +} + +function start_ews() +{ + trap 'error_handler ${LINENO} $?' ERR + + while : + do + date + echo "TASK: cleaning up old log files." + cd ~/win-ews-logs + rm -rf win-ews.old + mv win-ews.log win-ews.old + find ~/win-ews-logs -mtime +7 -exec rm -f {} \; + echo "TASK: Starting up" + cd ~/WebKit + echo "TASK: Cleaning WebKitBuild" + rm -rf WebKitBuild + date + echo "TASK: Cleaning up" + svn cleanup + date + echo "TASK: svn revert -R *" + svn revert -R * + date + echo "TASK: svn status" + svn status | grep "?" | awk '{print $2}' | xargs rm -rf + date + echo "TASK: webkit-patch clean" + ~/WebKit/Tools/Scripts/webkit-patch clean + date + echo "TASK: svn up --non-interactive" + svn up --non-interactive + date + echo "TASK: kill orphaned tasks" + taskkill.exe /f /im cl.exe + echo "TASK: webkit-patch win-ews" + ~/WebKit/Tools/Scripts/webkit-patch win-ews --bot-id=$BOT_ID --no-confirm --exit-after-iteration 10 + echo "TASK: kill old processes" + ~/WebKit/Tools/BuildSlaveSupport/kill-old-processes + done +} + +start_ews
\ No newline at end of file diff --git a/Tools/EWSTools/start-queue.sh b/Tools/EWSTools/start-queue.sh index 801de996f..c1d544de1 100755 --- a/Tools/EWSTools/start-queue.sh +++ b/Tools/EWSTools/start-queue.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012 Google Inc. All rights reserved. +# Copyright (c) 2013 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -11,9 +11,6 @@ # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -27,31 +24,63 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +USAGE="Usage: start-queue-loop.sh [OPTIONS] QUEUE_NAME BOT_ID RESET_AFTER_ITERATION + -r RESET_SCRIPT A bash script to be run every cycle just before the queue starts up again. + -p QUEUE_PARAMS Additional parameters to be given to the queue's execution command." + +while getopts "r:p:" FLAG +do + case $FLAG in + r) RESET_SCRIPT=$OPTARG;; + p) QUEUE_PARAMS=$OPTARG;; + esac +done + +shift $((OPTIND-1)) + +if [[ "$#" -ne 3 ]]; then + echo "$USAGE" + exit 1 +fi + QUEUE_NAME=$1 BOT_ID=$2 +RESET_AFTER_ITERATION=$3 cd /mnt/git/webkit-$QUEUE_NAME while : do - # This somewhat quirky sequence of steps seems to clear up all the broken - # git situations we've gotten ourself into in the past. - git clean -f # Remove any left-over layout test results, added files, etc. - git rebase --abort # If we got killed during a git rebase, we need to clean up. - git fetch origin # Avoid updating the working copy to a stale revision. - git checkout origin/master -f - git branch -D master - git checkout origin/master -b master - - # Most queues auto-update as part of their normal operation, but updating - # here makes sure that we get the latest version of the master process. - ./Tools/Scripts/update-webkit - - # test-webkitpy has code to remove orphaned .pyc files, so we - # run it before running webkit-patch to avoid stale .pyc files - # preventing webkit-patch from launching. - ./Tools/Scripts/test-webkitpy - - # We use --exit-after-iteration to pick up any changes to webkit-patch, including - # changes to the committers.py file. - ./Tools/Scripts/webkit-patch $QUEUE_NAME --bot-id=$BOT_ID --no-confirm --exit-after-iteration 10 + # This clears any temporary file leaks after running tests. + # Not the nicest solution but it will keep the queues running instead of + # filling up all remaining disk space. + # NOTE: This will clear any Chromium browser temporary files running on the + # machine as well. Not advised to run this while using Chrome. + find /tmp -name ".org.chromium.Chromium.*" | xargs rm -rf + + # This somewhat quirky sequence of steps seems to clear up all the broken + # git situations we've gotten ourself into in the past. + git clean -f -d # Remove any left-over layout test results, added files, etc. + git rebase --abort # If we got killed during a git rebase, we need to clean up. + git fetch origin # Avoid updating the working copy to a stale revision. + git checkout origin/master -f + git branch -D master + git checkout origin/master -b master + + # Most queues auto-update as part of their normal operation, but updating + # here makes sure that we get the latest version of the master process. + ./Tools/Scripts/update-webkit + + # test-webkitpy has code to remove orphaned .pyc files, so we + # run it before running webkit-patch to avoid stale .pyc files + # preventing webkit-patch from launching. + ./Tools/Scripts/test-webkitpy + + # Run the given reset script. + if [[ -n "$RESET_SCRIPT" ]]; then + bash "$RESET_SCRIPT" + fi + + # We use --exit-after-iteration to pick up any changes to webkit-patch, including + # changes to the contributors.json file. + ./Tools/Scripts/webkit-patch $QUEUE_NAME --bot-id=$BOT_ID --no-confirm --exit-after-iteration $RESET_AFTER_ITERATION $QUEUE_PARAMS done |