Qpid-1934 : Add start/stop script for broker monitoring.

Add runTest.sh that starts a given set of tests and then generates stat files for them
Updated monitor-broker to use $! to get top pid, rather than ps|grep
made runTests.sh take a file which contains a list of tests to run.
Addition of processTests:
   Creates a Result Package suitable for processing by process.sh / processAll.sh
Updated process.sh to correctly handle new broker_cpu.log date format and to have better error handling when there is no GC data
Update to monitor-broker to record the top rate so it can be used in processing
Update to process.sh to better display the CPU x axis using the time.
Update to processTests.py to use a regex to extract the GC time stamp.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk/qpid@787292 13f79535-47bb-0310-9956-ffa450edef68

3 files changed, 430 insertions, 0 deletions

diff --git a/java/perftests/bin/monitoring/monitor-broker.sh b/java/perftests/bin/monitoring/monitor-broker.sh
new file mode 100755
index 0000000000..39f4760114
--- /dev/null
+++ b/java/perftests/bin/monitoring/monitor-broker.sh
@@ -0,0 +1,200 @@
+#!/bin/bash
+#
+# This script starts a broker and then starts additional logging as required.
+#  *.pid files are generated in the LOG_DIR for later use by the stop-monitored-broker 
+#  script. 
+# 
+# Currently this process starts:
+#   - The broker with additional QPID_OPTS for gc logging
+#   - Top to monitoring the CPU usage
+#
+# Additional processes can be started and as long as they write a PID into LOG_DIR/*.pid
+# it will be shutdown with the stop script
+#
+
+#
+# Output the broker log file to aid problem diagnosis
+# then exit.
+#
+brokerFailExit()
+{
+   echo "Broker failed to start up."
+   cat $BROKER_LOG
+   exit 1 
+}
+
+showUsageExit()
+{
+  echo "Usage $0 <Path to Test Broker> <LOG DIR> <CPU Monitor Rate (s)> [Additional options to
+  pass to Qpid broker startup]"
+  exit 1
+}
+
+#
+# Perform 3 attempts to get the broker PID via ps and grep
+# if unable the output broker log and exit
+#
+getBrokerPID()
+{
+ attempts=3
+ ready=0
+ while [ $ready == 0 ] ; do
+ 
+  PID=`ps auxwww| grep java | grep Xloggc | awk '{print $2}'`
+
+  if [ ! $PID == 0 ] ; then
+   ready=1 
+  else  
+   attempts=$[ $attempts - 1 ]
+ 
+   if [ $attempts == 0 ] ; then
+     brokerFailExit
+   fi
+   
+   sleep 1
+  fi
+ done
+
+}
+
+
+#
+# Additional Check to ensure that the broker process
+# has correctly written 'Ready' to the log file.
+#
+checkBrokerStarted()
+{
+ attempts=3
+ ready=0
+ while [ $ready == 0 ] ; do
+ 
+  grep Ready $BROKER_LOG > /dev/null
+
+  if [ $? == 0 ] ; then
+   ready=1 
+  else  
+   attempts=$[ $attempts - 1 ]
+ 
+   if [ $attempts == 0 ] ; then
+     brokerFailExit
+   fi
+   
+   echo "Broker not ready sleeping 1s"
+   sleep 1
+  fi
+ done
+}
+
+#
+# Command Line setup
+#
+
+# Ensure we have minimum of three arguments
+if [[ $# > 2 ]] ; then
+  BROKER_VERSION=$1 
+  LOG_DIR=$2
+  CPU_MONITOR_RATE=$3
+  # Remove these arguments from the $@ variable
+  shift 
+  shift 
+  shift
+else
+  # If we have no arguments then use these as the default  
+  CPU_MONITOR_RATE=0.5
+  LOG_DIR=$QPID_WORK/logging
+  BROKER_VERSION=qpid-0.5
+fi
+
+
+#
+# Check the specified broker is reachable
+# it it is not the log and show usage
+#
+if [ ! -d $BROKER_VERSION ] ; then
+  echo "Broker not available at: $BROKER_VERSION"
+  showUsageExit
+fi
+
+#
+# Check to see if we have an absolute path for logging
+#
+logStart=`echo $LOG_DIR|cut -c 1`
+
+
+#
+# If we don't have an absolute path then add the current 
+# directory path to the start.
+#
+if [[ $logStart != '/' ]] ; then
+ echo -n "$LOG_DIR is not absolute, using "
+ LOG_DIR=`pwd`/$LOG_DIR
+ echo $LOG_DIR
+fi
+
+#
+# Validate that the directory does not exist
+#  - this is so we can guarrantee a clean run.
+# If it does exit then log and show usage
+#
+if [ -d $LOG_DIR ] ; then
+  echo "Log directory already exists : $LOG_DIR"
+  showUsageExit
+fi
+
+#
+# Create the logging directory
+#
+mkdir -p $LOG_DIR
+
+#
+# Variable for broker log 
+#
+BROKER_LOG=$LOG_DIR/broker.log
+
+# Variable to hold broker PID
+PID=0
+
+export QPID_OPTS="-Xloggc:$LOG_DIR/gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" 
+
+#
+# Start Qpid Broker
+#
+echo "Starting Broker : $BROKER_VERSION"
+pushd $BROKER_VERSION/bin > /dev/null
+./qpid-server $@ 2> $BROKER_LOG >&2 &
+popd > /dev/null
+
+# Wait and check startup was ok
+echo "Waiting for broker startup"
+getBrokerPID
+
+checkBrokerStarted
+
+echo $PID > $LOG_DIR/broker.pid
+
+# 
+# Start CPU Monitoring via TOP
+#
+echo "Starting CPU Monitor at RATE:$CPU_MONITOR_RATE on $SERVER1"
+pushd $LOG_DIR > /dev/null
+
+echo $CPU_MONITOR_RATE > top.rate 
+
+top -d $CPU_MONITOR_RATE -S -c -p $PID -b > broker_cpu.log &
+
+#
+# Get top pid using $!
+#
+echo $! > $LOG_DIR/top.pid
+
+popd > /dev/null
+
+
+#
+# Generate Stat files
+#
+echo "Generating Stat data"
+stat $BROKER_LOG > $BROKER_LOG.stat
+stat $LOG_DIR/broker_cpu.log > $LOG_DIR/broker_cpu.log.stat
+stat $LOG_DIR/gc.log > $LOG_DIR/gc.log.stat
+
diff --git a/java/perftests/bin/monitoring/runTests.sh b/java/perftests/bin/monitoring/runTests.sh
new file mode 100755
index 0000000000..4ba5040ccc
--- /dev/null
+++ b/java/perftests/bin/monitoring/runTests.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+#
+# Run specified performance tests and 
+# gather details about the test run
+#
+
+
+runTest()
+{
+ echo "$@"
+ echo "$@ --csv -o $LOG_DIR" >> $LOG_DIR/TestRun.log 2>&1
+ ./$@ --csv -o $LOG_DIR >> $LOG_DIR/TestRun.log 2>&1
+}
+
+showUsageExit()
+{
+  echo "Usage $0 <Path to Test Pack> <LOG DIR> <TEST LIST FILE>"
+  exit 1
+}
+
+# Ensure we have minimum of three arguments
+if [[ $# > 2 ]] ; then
+  TEST_VERSION=$1 
+  LOG_DIR=$2
+  TEST_LIST=$3
+  # Remove these arguments from the $@ variable
+  shift 
+  shift
+  shift
+else
+  showUsageExit
+fi
+
+#
+# Check the specified broker is reachable
+# it it is not the log and show usage
+#
+if [ ! -d $TEST_VERSION ] ; then
+  echo "Tests not available at: $TEST_VERSION"
+  showUsageExit
+fi
+
+
+#
+# Check to see if we have an absolute path for logging
+#
+logStart=`echo $LOG_DIR|cut -c 1`
+
+#
+# If we don't have an absolute path then add the current 
+# directory path to the start.
+#
+if [[ $logStart != '/' ]] ; then
+ echo -n "$LOG_DIR is not absolute, using "
+ LOG_DIR=`pwd`/$LOG_DIR
+ echo $LOG_DIR
+fi
+
+#
+# Validate that the directory does not exist
+#  - this is so we can guarrantee a clean run.
+# If it does exit then log and show usage
+#
+if [ -d $LOG_DIR ] ; then
+  echo "Log directory already exists : $LOG_DIR"
+  showUsageExit
+fi
+
+#
+# Check to see if we have an absolute path for test list
+#
+testListStart=`echo $TEST_LIST|cut -c 1`
+
+#
+# If we don't have an absolute path then add the current 
+# directory path to the start.
+#
+if [[ $testListStart != '/' ]] ; then
+ echo -n "$TEST_LIST is not absolute, using "
+ TEST_LIST=`pwd`/$TEST_LIST
+ echo $TEST_LIST
+fi
+
+#
+# Validate that the directory does not exist
+#  - this is so we can guarrantee a clean run.
+# If it does exit then log and show usage
+#
+# -r Check file exists and is readable
+if [ ! -r $TEST_LIST ] ; then
+  echo "Test file is not readable : $TEST_LIST"
+  showUsageExit
+fi
+
+
+
+#
+# Create the logging directory
+#
+mkdir -p $LOG_DIR
+
+
+
+echo "Starting Test Run in : $TEST_VERSION"
+pushd $TEST_VERSION/bin > /dev/null
+
+#
+# Run tests
+#
+
+
+while read testCommand
+do
+ runTest $testCommand
+done < "$TEST_LIST"
+
+
+popd > /dev/null
+
+
+#
+# Generate Stat files
+#
+echo "Generating Stat data"
+for file in `find $LOG_DIR -name "*.csv"` ; do
+  stat $file > $file.stat
+done
diff --git a/java/perftests/bin/monitoring/stop-monitored-broker.sh b/java/perftests/bin/monitoring/stop-monitored-broker.sh
new file mode 100755
index 0000000000..2bd1d1771a
--- /dev/null
+++ b/java/perftests/bin/monitoring/stop-monitored-broker.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+#
+# Script to stop the running of a monitored broker
+# and the associated monitoring processes.
+#
+# Looks in the specifed directory for pid files and 
+# stops those proceses
+#
+
+
+usage()
+{
+  echo "Usage: $0 <LOG_DIR>"
+}
+
+#
+# Attempt to gracefully kill processs
+#
+stopRun()
+{
+  kill $PIDS
+}
+
+
+#
+# Forcibly stop processes
+#
+forceStopRun()
+{
+  kill -9 $PIDS
+}
+
+#
+# Show usage if we are not started correctly
+#
+if [ $# != 1 ] ; then
+ usage
+ exit 1
+fi
+
+LOG_DIR=$1
+
+
+PIDS=`cat $LOG_DIR/*.pid`
+
+echo "Preparing to stop:"
+#
+# Escape the path so we can use sed to remove it.
+#
+path=`echo $LOG_DIR|sed -e s/\\\//\\\\\\\\\\\//g`
+
+for i in `ls $LOG_DIR/*.pid` ; do
+  # Remove path from pid item then remove any final '/' 
+  echo $i|cut -d '.' -f 1| sed -e s/$path// |tr '/' ' '
+done
+
+status=`ps $PIDS |wc -l`
+
+if [ $status == 1 ] ; then
+ echo "Processes do not appear to be running."
+ echo "Have they already been stopped?"
+ exit 0
+fi
+
+attempts=3
+
+while [ ! $status == 1 ] ; do
+ stopRun
+ sleep 1
+ status=`ps $PIDS |wc -l`
+ 
+  if [ $status == 1 ] ; then
+    echo "Done"
+    exit 0
+  else  
+   attempts=$[ $attempts - 1 ]
+ 
+   if [ $attempts == 0 ] ; then
+     break
+   fi
+   
+  echo "Sleeping as processes not stopped"
+  sleep 2
+
+  fi
+done
+
+# If we haven't been able to kill the processes then 
+# forcibly do it
+if [ ! $status == 1 ] ; then
+ forceStopRun
+ sleep 1
+ status=`ps $PIDS |wc -l`
+ 
+  if [ $status == 1 ] ; then
+    echo "Done"
+  else
+    echo "Stop failed"
+    exit 1
+  fi 
+else
+ echo "Done"
+fi