From f6c31f823840fbb37ce68e730af4fba2b750884c Mon Sep 17 00:00:00 2001 From: Robert Gemmell Date: Wed, 4 Apr 2012 16:15:04 +0000 Subject: QPID-3929: remove old perftests + integrationtests + junit-toolkit modules and associated files git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk/qpid@1309476 13f79535-47bb-0310-9956-ffa450edef68 --- java/perftests/RunningPerformanceTests.txt | 141 -- java/perftests/bin/monitoring/monitor-broker.sh | 221 --- java/perftests/bin/monitoring/runTests.sh | 148 -- .../bin/monitoring/stop-monitored-broker.sh | 124 -- java/perftests/bin/processing/process.sh | 388 ----- java/perftests/bin/processing/processAll.sh | 58 - java/perftests/bin/processing/processTests.py | 850 --------- java/perftests/bin/run_many.sh | 30 - java/perftests/bin/topicListener.sh | 32 - java/perftests/bin/topicPublisher.sh | 31 - java/perftests/build.xml | 339 ---- java/perftests/dist-zip.xml | 45 - java/perftests/etc/jndi/activemq.properties | 20 - java/perftests/etc/jndi/failovertest.properties | 19 - java/perftests/etc/jndi/perftests.properties | 19 - java/perftests/etc/jndi/swiftmq.properties | 20 - java/perftests/etc/perftests.log4j | 46 - java/perftests/etc/scripts/CTQ-Qpid-1.sh | 21 - java/perftests/etc/scripts/CTQ-Qpid-2.sh | 21 - java/perftests/etc/scripts/CTQ-Qpid-3.sh | 21 - java/perftests/etc/scripts/CTQ-Qpid-4.sh | 21 - java/perftests/etc/scripts/CTQ-Qpid-5.sh | 21 - java/perftests/etc/scripts/CTQ-Qpid-6.sh | 21 - java/perftests/etc/scripts/Connections.sh | 20 - java/perftests/etc/scripts/JobQueue.sh | 20 - java/perftests/etc/scripts/Latency.sh | 21 - java/perftests/etc/scripts/MessageSize.sh | 20 - java/perftests/etc/scripts/PT-Qpid-13.sh | 42 - java/perftests/etc/scripts/PT-Qpid-14.sh | 41 - java/perftests/etc/scripts/Reliability.sh | 20 - java/perftests/etc/scripts/RunAll.sh | 23 - java/perftests/etc/scripts/RunCore.sh | 22 - java/perftests/etc/scripts/Test-ActiveMQ.sh | 32 - java/perftests/etc/scripts/Test-SwiftMQ.sh | 30 - java/perftests/etc/scripts/Throughput.sh | 20 - java/perftests/etc/scripts/drainBroker.sh | 41 - java/perftests/etc/scripts/extractResults.sh | 34 - .../etc/scripts/extractThroughputResults.sh | 51 - java/perftests/etc/scripts/fillBroker.sh | 41 - java/perftests/etc/scripts/sendAndWaitClient.sh | 22 - java/perftests/etc/scripts/testWithPreFill.sh | 41 - java/perftests/generate-scripts | 74 - java/perftests/jar-with-dependencies.xml | 91 - java/perftests/scripts.xml | 328 ---- .../qpid/client/message/TestMessageFactory.java | 117 -- .../config/AMQConnectionFactoryInitialiser.java | 47 - .../org/apache/qpid/config/AbstractConfig.java | 69 - .../qpid/config/ConnectionFactoryInitialiser.java | 29 - .../java/org/apache/qpid/config/Connector.java | 43 - .../org/apache/qpid/config/ConnectorConfig.java | 28 - .../config/JBossConnectionFactoryInitialiser.java | 112 -- .../main/java/org/apache/qpid/oldtopic/Config.java | 243 --- .../java/org/apache/qpid/oldtopic/Listener.java | 141 -- .../org/apache/qpid/oldtopic/MessageFactory.java | 153 -- .../java/org/apache/qpid/oldtopic/Publisher.java | 178 -- .../org/apache/qpid/ping/PingAsyncTestPerf.java | 323 ---- .../main/java/org/apache/qpid/ping/PingClient.java | 112 -- .../org/apache/qpid/ping/PingDurableClient.java | 452 ----- .../org/apache/qpid/ping/PingLatencyTestPerf.java | 311 ---- .../org/apache/qpid/ping/PingSendOnlyClient.java | 93 - .../java/org/apache/qpid/ping/PingTestPerf.java | 281 --- .../apache/qpid/requestreply/PingPongBouncer.java | 453 ----- .../apache/qpid/requestreply/PingPongProducer.java | 1818 -------------------- .../apache/qpid/requestreply/PingPongTestPerf.java | 251 --- .../main/java/org/apache/qpid/topic/Config.java | 326 ---- .../main/java/org/apache/qpid/topic/Listener.java | 303 ---- .../java/org/apache/qpid/topic/MessageFactory.java | 157 -- .../main/java/org/apache/qpid/topic/Publisher.java | 186 -- .../TopicWithSelectorsTransientVolumeTest.java | 344 ---- .../apache/qpid/topic/topicselectors.properties | 24 - 70 files changed, 10285 deletions(-) delete mode 100644 java/perftests/RunningPerformanceTests.txt delete mode 100755 java/perftests/bin/monitoring/monitor-broker.sh delete mode 100755 java/perftests/bin/monitoring/runTests.sh delete mode 100755 java/perftests/bin/monitoring/stop-monitored-broker.sh delete mode 100755 java/perftests/bin/processing/process.sh delete mode 100755 java/perftests/bin/processing/processAll.sh delete mode 100755 java/perftests/bin/processing/processTests.py delete mode 100755 java/perftests/bin/run_many.sh delete mode 100755 java/perftests/bin/topicListener.sh delete mode 100755 java/perftests/bin/topicPublisher.sh delete mode 100644 java/perftests/build.xml delete mode 100644 java/perftests/dist-zip.xml delete mode 100644 java/perftests/etc/jndi/activemq.properties delete mode 100644 java/perftests/etc/jndi/failovertest.properties delete mode 100644 java/perftests/etc/jndi/perftests.properties delete mode 100644 java/perftests/etc/jndi/swiftmq.properties delete mode 100644 java/perftests/etc/perftests.log4j delete mode 100755 java/perftests/etc/scripts/CTQ-Qpid-1.sh delete mode 100755 java/perftests/etc/scripts/CTQ-Qpid-2.sh delete mode 100755 java/perftests/etc/scripts/CTQ-Qpid-3.sh delete mode 100755 java/perftests/etc/scripts/CTQ-Qpid-4.sh delete mode 100755 java/perftests/etc/scripts/CTQ-Qpid-5.sh delete mode 100755 java/perftests/etc/scripts/CTQ-Qpid-6.sh delete mode 100755 java/perftests/etc/scripts/Connections.sh delete mode 100755 java/perftests/etc/scripts/JobQueue.sh delete mode 100755 java/perftests/etc/scripts/Latency.sh delete mode 100755 java/perftests/etc/scripts/MessageSize.sh delete mode 100755 java/perftests/etc/scripts/PT-Qpid-13.sh delete mode 100755 java/perftests/etc/scripts/PT-Qpid-14.sh delete mode 100755 java/perftests/etc/scripts/Reliability.sh delete mode 100755 java/perftests/etc/scripts/RunAll.sh delete mode 100755 java/perftests/etc/scripts/RunCore.sh delete mode 100644 java/perftests/etc/scripts/Test-ActiveMQ.sh delete mode 100644 java/perftests/etc/scripts/Test-SwiftMQ.sh delete mode 100755 java/perftests/etc/scripts/Throughput.sh delete mode 100755 java/perftests/etc/scripts/drainBroker.sh delete mode 100755 java/perftests/etc/scripts/extractResults.sh delete mode 100755 java/perftests/etc/scripts/extractThroughputResults.sh delete mode 100755 java/perftests/etc/scripts/fillBroker.sh delete mode 100755 java/perftests/etc/scripts/sendAndWaitClient.sh delete mode 100755 java/perftests/etc/scripts/testWithPreFill.sh delete mode 100644 java/perftests/generate-scripts delete mode 100644 java/perftests/jar-with-dependencies.xml delete mode 100644 java/perftests/scripts.xml delete mode 100644 java/perftests/src/main/java/org/apache/qpid/client/message/TestMessageFactory.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/config/AMQConnectionFactoryInitialiser.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/config/AbstractConfig.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/config/ConnectionFactoryInitialiser.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/config/Connector.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/config/ConnectorConfig.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/config/JBossConnectionFactoryInitialiser.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/oldtopic/Config.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/oldtopic/Listener.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/oldtopic/MessageFactory.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/oldtopic/Publisher.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/ping/PingAsyncTestPerf.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/ping/PingClient.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/ping/PingDurableClient.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/ping/PingLatencyTestPerf.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/ping/PingSendOnlyClient.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/ping/PingTestPerf.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/requestreply/PingPongBouncer.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/requestreply/PingPongProducer.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/requestreply/PingPongTestPerf.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/topic/Config.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/topic/Listener.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/topic/MessageFactory.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/topic/Publisher.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/topic/TopicWithSelectorsTransientVolumeTest.java delete mode 100644 java/perftests/src/main/java/org/apache/qpid/topic/topicselectors.properties (limited to 'java/perftests') diff --git a/java/perftests/RunningPerformanceTests.txt b/java/perftests/RunningPerformanceTests.txt deleted file mode 100644 index 2bc9220774..0000000000 --- a/java/perftests/RunningPerformanceTests.txt +++ /dev/null @@ -1,141 +0,0 @@ -The Performance Tests -------------------------- - -Building the Tests (Only develoeprs need to know how to do this). ------------------------------------------------------------------ - -The performance tests are compiled as part of the Maven build by default, but the performance test scripts are not. There is also an additional step to perform, that generates a convenient Jar file containing all of the test dependencies, to simplify invoking Java with a very long command line. The steps to build the performance test suite are: - - 1. Cd to the /java/perftests directory. - 2. Execute: mvn org.apache.qpid:junit-toolkit-maven-plugin:tkscriptgen (this generates the scripts). - 3. Execute: mvn assembly:assembly - -The assembly:assembly step generates a Jar with all the dependecies in it in a file name ending with -all-test-deps.jar, which contains the client code and all its dependencies, plus JUnit and the toolkit test runners. The generated scripts expect to find the jar in the current directory. You can Cd to the /target directory and run the scripts from there. The assembly:assembly step also outputs some archives that contain all the scripts and required Jar for convenient shipping of the test suite. Unpack this anywhere you like and run the tests from there. - -Running the Tests ------------------ - -All the performance tests are run through shell scripts, that have been configured with parameters set up in the pom.xml file. You can override any of these parameters on the command line. It is also possible to pass parameters through the script to the JVM that is running the test. For example to change the heap size you might do: - -./Ping-Once.sh -java:Xmx1024M - -The tests have all been set up to accept a single integer 'size' parameter, passed to the JUnit TestCase for the test, through the JUnit Toolkit asymptotic test case extension. The 'size' parameter is always interpreted in the performance tests as standing for the number of messages to send per test method invocation. Therefore, in the results of the test the throughput of messages is equal to the number of test method invocations times the 'size' divided by the time taken. - -** TODO: Change this, use seconds not millis. - -Test timing results are output to .csv files, which can easily be imported into a spreadsheet for graphing and analysis. The timings in this file are always given in milliseconds, which may be a bit confusing and should really be changed to seconds. - -The JUnit Toolkit provides a framework for controlling how long tests are run for, how many are run at once and what the 'size' parameter is, which is general enough to handle a wide variety of performance tests. Here is the documentation from the TKTestRunner class that explains what its command line parameters are: - - ... - * TKTestRunner extends {@link junit.textui.TestRunner} with the ability to run tests multiple times, to execute a test - * simultaneously using many threads, to put a delay between test runs and adds support for tests that take integer - * parameters that can be 'stepped' through on multiple test runs. These features can be accessed by using this class - * as an entry point and passing command line arguments to specify which features to use: - * - * 
- * -w ms       The number of milliseconds between invocations of test cases.
- * -c pattern  The number of tests to run concurrently.
- * -r num      The number of times to repeat each test.
- * -d duration The length of time to run the tests for.
- * -t name     The name of the test case to execute.
- * -s pattern  The size parameter to run tests with.
- * -o dir      The name of the directory to output test timings to.
- * -v          Verbose mode.
- *
- * - *

The pattern arguments are of the form [lowest(, ...)(, highest)](,sample=s)(,exp), where round brackets - * enclose optional values. Using this pattern form it is possible to specify a single value, a range of values divided - * into s samples, a range of values divided into s samples but distributed exponentially, or a fixed set of samples. - * - *

The duration arguments are of the form dD(:hH)(:mM)(:sS), where round brackets enclose optional values. - * - *

Here are some examples: - * - *

- * 
 -c [10,20,30,40,50]
Runs the test with 10,20,...,50 threads. - * 
 -s [1,100],samples=10
- * 		Runs the test with ten different size parameters evenly spaced between 1 and 100. - * 
 -s [1,1000000],samples=10,exp
- * 		Runs the test with ten different size parameters exponentially spaced between 1 and 1000000. - * 
 -r 10
Runs each test ten times. - * 
 -d 10H
Runs the test repeatedly for 10 hours. - * 
 -d 1M, -r 10
- * 		Runs the test repeatedly for 1 minute but only takes a timing sample every 10 test runs. - * 
 -r 10, -c [1, 5, 10, 50], -s [100, 1000, 10000]
- * 		Runs 12 test cycles (4 concurrency samples * 3 size sample), with 10 repeats each. In total the test - * will be run 199 times (3 + 15 + 30 + 150) - *
- ... - -The specific performance test cases for QPid are implemented as extensions to JUnit TestCase (asymptotic test cases), and also accept a large number of different parameters to control the characteristics of the test. The are passed into the test scripts as name=value pairs. Here is the documentation from the PingPongProducer class that explains what the available parameters and default values are: - - ... - *

This ping tool accepts a vast number of configuration options, all of which are passed in to the constructor. It - * can ping topics or queues; ping multiple destinations; do persistent pings; send messages of any size; do pings within - * transactions; control the number of pings to send in each transaction; limit its sending rate; and perform failover - * testing. A complete list of accepted parameters, default values and comments on their usage is provided here: - * - *

- *
Parameters
Parameter Default Comments - *
messageSize 0 Message size in bytes. Not including any headers. - *
destinationName ping The root name to use to generate destination names to ping. - *
persistent false Determines whether peristent delivery is used. - *
transacted false Determines whether messages are sent/received in transactions. - *
broker tcp://localhost:5672 Determines the broker to connect to. - *
virtualHost test Determines the virtual host to send all ping over. - *
rate 0 The maximum rate (in hertz) to send messages at. 0 means no limit. - *
verbose false The verbose flag for debugging. Prints to console on every message. - *
pubsub false Whether to ping topics or queues. Uses p2p by default. - *
failAfterCommit false Whether to prompt user to kill broker after a commit batch. - *
failBeforeCommit false Whether to prompt user to kill broker before a commit batch. - *
failAfterSend false Whether to prompt user to kill broker after a send. - *
failBeforeSend false Whether to prompt user to kill broker before a send. - *
failOnce true Whether to prompt for failover only once. - *
username guest The username to access the broker with. - *
password guest The password to access the broker with. - *
selector null Not used. Defines a message selector to filter pings with. - *
destinationCount 1 The number of receivers listening to the pings. - *
timeout 30000 In milliseconds. The timeout to stop waiting for replies. - *
commitBatchSize 1 The number of messages per transaction in transactional mode. - *
uniqueDests true Whether each receiver only listens to one ping destination or all. - *
durableDests false Whether or not durable destinations are used. - *
ackMode AUTO_ACK The message acknowledgement mode. Possible values are: - * 0 - SESSION_TRANSACTED - * 1 - AUTO_ACKNOWLEDGE - * 2 - CLIENT_ACKNOWLEDGE - * 3 - DUPS_OK_ACKNOWLEDGE - * 257 - NO_ACKNOWLEDGE - * 258 - PRE_ACKNOWLEDGE - *
maxPending 0 The maximum size in bytes, of messages sent but not yet received. - * Limits the volume of messages currently buffered on the client - * or broker. Can help scale test clients by limiting amount of buffered - * data to avoid out of memory errors. - *
- ... - -The most common test case to run is implemented in the class PingAsyncTestPerf, which sends and recieves messages simultaneously. This class uses a PingPongProdicer to do its sending and receiving, and wraps it in a suitable way to make it callable through the extended JUnit test runner. This class also accpets another parameter "batchSize" with a default of "1000". This tells the test how many messages to send before stopping sending and waiting for them all to come back. The actual value entered does not matter too much, but typically values larger than 1000 are used to ensure that there is a reasonable opportunity for simultaneous sending and receiving, and less than 10000 to ensure that each test method invocation does not go on for too long. - -The test script parameters can all be seen in the pom.xml file. A three letter code is used on the test scripts, first letter P or T for persistent or transient, second letter Q or T for queue (p2p) or topic (pub/sub), third letter R for reliability tests, C for client scaling tests, M for message size tests.Typically tests run and sample their results for 10 minutes, to get a reasonable measurement of a broker running under a steady load. The tests as configured do not measure 'burst' performance. - -The reliability/burn in tests, test the broker running at slightly below its maximum throughput for a period of 24 hours. Their purpose is to check that the broker remains stable under load for a reasonable duration, in order to provide some confidence in the long-term stability of its process. These tests are intended to be run as a two step process. The first two tests run for 10 minutes and are used to asses the broker throughput for the test. The output from these tests are to be fed into the rate limiter for the second set of tests, so that the broker may be set up to run at slightly below its maximum throughput for the 24 hour duration. It is suggested that 90% of the rate achieved by the first two tests should be used for this. - -The client scaling tests are split into two sub-sections. The first section tests the performance of increasing numbers of client connections, each sending at a fixed rate. The purpose of this is to determine the brokers saturation load, and to evaluate how its performance degrades uder higher loads. The second section varies the fan-out or fan-in ratio of the number of sending clients to receving clients. This is primarily intended to test the pubsub messaging model, but the tests are also run in p2p mode (with each message being received by one consumer), for completeness and to provide a comparison with the pubsub performance. - -The message size scaling tests, examine the brokers performance with different message payload sizes. The purpose of these tests is to evaluate where the broker process switches from being an io-bound to a cpu-bound process (if at all). The expected model is that the amount of CPU processing the broker has to carry out depends largely on the number of messages, and not on their size, because it carries out de-framing and routing for each message header but just copies payloads in-place or in a tight instruction loop. Therefore large message should be io-bound and a constant data rate through the broker should be seen for messages larger than the io/cpu threshold. Small messages require more processing so a constant message rate should be seen for message smaller than the io/cpu threshold. If the broker implementation is extremely efficient the threshold may dissapear altogether and the broker will be purely io-bound. -The final variation, which is applied to all tests, is to run a transactional and non-transactional version of each. Messages are always batched into transactions of 100 messages each. - -Running the entire test suite can take some time, in particular their are about 4 24 hour burn-in tests. There are also 8 30 minute client scaling ramp up tests. If you want to run the test for a short time, to skim test that they work on your environment a command line like the following is usefull: - -> find . -name '*.sh' -exec {} -d10S \; - -If you want to run just a sub-set of the tests, you can use variations of the above command line. For example, to run just the message size tests using persistent p2p messaging do: - -> find . -name 'PPM-*.sh' -exec {} \; - -and so on. - -Interpreting the Results ------------------------- - -TODO: Explain what the results are expected to show and how to look for it. What should be graphed to get a visualization of the broker performance. How to turn the measurements into a description of the performance 'envelope'. \ No newline at end of file diff --git a/java/perftests/bin/monitoring/monitor-broker.sh b/java/perftests/bin/monitoring/monitor-broker.sh deleted file mode 100755 index 27c8ff25f6..0000000000 --- a/java/perftests/bin/monitoring/monitor-broker.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# This script starts a broker and then starts additional logging as required. -# *.pid files are generated in the LOG_DIR for later use by the stop-monitored-broker -# script. -# -# Currently this process starts: -# - The broker with additional QPID_OPTS for gc logging -# - Top to monitoring the CPU usage -# -# Additional processes can be started and as long as they write a PID into LOG_DIR/*.pid -# it will be shutdown with the stop script -# - -# -# Output the broker log file to aid problem diagnosis -# then exit. -# -brokerFailExit() -{ - echo "Broker failed to start up." - cat $BROKER_LOG - exit 1 -} - -showUsageExit() -{ - echo "Usage $0 [Additional options to - pass to Qpid broker startup]" - exit 1 -} - -# -# Perform 3 attempts to get the broker PID via ps and grep -# if unable the output broker log and exit -# -getBrokerPID() -{ - attempts=3 - ready=0 - while [ $ready == 0 ] ; do - - PID=`ps auxwww| grep java | grep Xloggc | awk '{print $2}'` - - if [ ! $PID == 0 ] ; then - ready=1 - else - attempts=$[ $attempts - 1 ] - - if [ $attempts == 0 ] ; then - brokerFailExit - fi - - sleep 1 - fi - done - -} - - -# -# Additional Check to ensure that the broker process -# has correctly written 'Ready' to the log file. -# -checkBrokerStarted() -{ - attempts=3 - ready=0 - while [ $ready == 0 ] ; do - - grep Ready $BROKER_LOG > /dev/null - - if [ $? == 0 ] ; then - ready=1 - else - attempts=$[ $attempts - 1 ] - - if [ $attempts == 0 ] ; then - brokerFailExit - fi - - echo "Broker not ready sleeping 1s" - sleep 1 - fi - done -} - -# -# Command Line setup -# - -# Ensure we have minimum of three arguments -if [[ $# > 2 ]] ; then - BROKER_VERSION=$1 - LOG_DIR=$2 - CPU_MONITOR_RATE=$3 - # Remove these arguments from the $@ variable - shift - shift - shift -else - # If we have no arguments then use these as the default - CPU_MONITOR_RATE=0.5 - LOG_DIR=$QPID_WORK/logging - BROKER_VERSION=qpid-0.5 -fi - - -# -# Check the specified broker is reachable -# it it is not the log and show usage -# -if [ ! -d $BROKER_VERSION ] ; then - echo "Broker not available at: $BROKER_VERSION" - showUsageExit -fi - -# -# Check to see if we have an absolute path for logging -# -logStart=`echo $LOG_DIR|cut -c 1` - - -# -# If we don't have an absolute path then add the current -# directory path to the start. -# -if [[ $logStart != '/' ]] ; then - echo -n "$LOG_DIR is not absolute, using " - LOG_DIR=`pwd`/$LOG_DIR - echo $LOG_DIR -fi - -# -# Validate that the directory does not exist -# - this is so we can guarrantee a clean run. -# If it does exit then log and show usage -# -if [ -d $LOG_DIR ] ; then - echo "Log directory already exists : $LOG_DIR" - showUsageExit -fi - -# -# Create the logging directory -# -mkdir -p $LOG_DIR - -# -# Variable for broker log -# -BROKER_LOG=$LOG_DIR/broker.log - -# Variable to hold broker PID -PID=0 - -export QPID_OPTS="-Xloggc:$LOG_DIR/gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" - -# -# Start Qpid Broker -# -echo "Starting Broker : $BROKER_VERSION" -pushd $BROKER_VERSION/bin > /dev/null -./qpid-server $@ 2> $BROKER_LOG >&2 & -popd > /dev/null - -# Wait and check startup was ok -echo "Waiting for broker startup" -getBrokerPID - -checkBrokerStarted - -echo $PID > $LOG_DIR/broker.pid - -# -# Start CPU Monitoring via TOP -# -echo "Starting CPU Monitor at RATE:$CPU_MONITOR_RATE on $SERVER1" -pushd $LOG_DIR > /dev/null - -echo $CPU_MONITOR_RATE > top.rate - -top -d $CPU_MONITOR_RATE -S -c -p $PID -b > broker_cpu.log & - -# -# Get top pid using $! -# -echo $! > $LOG_DIR/top.pid - -popd > /dev/null - - -# -# Generate Stat files -# -echo "Generating Stat data" -stat $BROKER_LOG > $BROKER_LOG.stat -stat $LOG_DIR/broker_cpu.log > $LOG_DIR/broker_cpu.log.stat -stat $LOG_DIR/gc.log > $LOG_DIR/gc.log.stat - diff --git a/java/perftests/bin/monitoring/runTests.sh b/java/perftests/bin/monitoring/runTests.sh deleted file mode 100755 index 676db0d4fa..0000000000 --- a/java/perftests/bin/monitoring/runTests.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# Run specified performance tests and -# gather details about the test run -# - - -runTest() -{ - echo "$@" - echo "$@ --csv -o $LOG_DIR" >> $LOG_DIR/TestRun.log 2>&1 - ./$@ --csv -o $LOG_DIR >> $LOG_DIR/TestRun.log 2>&1 -} - -showUsageExit() -{ - echo "Usage $0 " - exit 1 -} - -# Ensure we have minimum of three arguments -if [[ $# > 2 ]] ; then - TEST_VERSION=$1 - LOG_DIR=$2 - TEST_LIST=$3 - # Remove these arguments from the $@ variable - shift - shift - shift -else - showUsageExit -fi - -# -# Check the specified broker is reachable -# it it is not the log and show usage -# -if [ ! -d $TEST_VERSION ] ; then - echo "Tests not available at: $TEST_VERSION" - showUsageExit -fi - - -# -# Check to see if we have an absolute path for logging -# -logStart=`echo $LOG_DIR|cut -c 1` - -# -# If we don't have an absolute path then add the current -# directory path to the start. -# -if [[ $logStart != '/' ]] ; then - echo -n "$LOG_DIR is not absolute, using " - LOG_DIR=`pwd`/$LOG_DIR - echo $LOG_DIR -fi - -# -# Validate that the directory does not exist -# - this is so we can guarrantee a clean run. -# If it does exit then log and show usage -# -if [ -d $LOG_DIR ] ; then - echo "Log directory already exists : $LOG_DIR" - showUsageExit -fi - -# -# Check to see if we have an absolute path for test list -# -testListStart=`echo $TEST_LIST|cut -c 1` - -# -# If we don't have an absolute path then add the current -# directory path to the start. -# -if [[ $testListStart != '/' ]] ; then - echo -n "$TEST_LIST is not absolute, using " - TEST_LIST=`pwd`/$TEST_LIST - echo $TEST_LIST -fi - -# -# Validate that the directory does not exist -# - this is so we can guarrantee a clean run. -# If it does exit then log and show usage -# -# -r Check file exists and is readable -if [ ! -r $TEST_LIST ] ; then - echo "Test file is not readable : $TEST_LIST" - showUsageExit -fi - - - -# -# Create the logging directory -# -mkdir -p $LOG_DIR - - - -echo "Starting Test Run in : $TEST_VERSION" -pushd $TEST_VERSION/bin > /dev/null - -# -# Run tests -# - - -while read testCommand -do - runTest $testCommand -done < "$TEST_LIST" - - -popd > /dev/null - - -# -# Generate Stat files -# -echo "Generating Stat data" -for file in `find $LOG_DIR -name "*.csv"` ; do - stat $file > $file.stat -done diff --git a/java/perftests/bin/monitoring/stop-monitored-broker.sh b/java/perftests/bin/monitoring/stop-monitored-broker.sh deleted file mode 100755 index ad882b0bea..0000000000 --- a/java/perftests/bin/monitoring/stop-monitored-broker.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# Script to stop the running of a monitored broker -# and the associated monitoring processes. -# -# Looks in the specifed directory for pid files and -# stops those proceses -# - - -usage() -{ - echo "Usage: $0 " -} - -# -# Attempt to gracefully kill processs -# -stopRun() -{ - kill $PIDS -} - - -# -# Forcibly stop processes -# -forceStopRun() -{ - kill -9 $PIDS -} - -# -# Show usage if we are not started correctly -# -if [ $# != 1 ] ; then - usage - exit 1 -fi - -LOG_DIR=$1 - - -PIDS=`cat $LOG_DIR/*.pid` - -echo "Preparing to stop:" -# -# Escape the path so we can use sed to remove it. -# -path=`echo $LOG_DIR|sed -e s/\\\//\\\\\\\\\\\//g` - -for i in `ls $LOG_DIR/*.pid` ; do - # Remove path from pid item then remove any final '/' - echo $i|cut -d '.' -f 1| sed -e s/$path// |tr '/' ' ' -done - -status=`ps $PIDS |wc -l` - -if [ $status == 1 ] ; then - echo "Processes do not appear to be running." - echo "Have they already been stopped?" - exit 0 -fi - -attempts=3 - -while [ ! $status == 1 ] ; do - stopRun - sleep 1 - status=`ps $PIDS |wc -l` - - if [ $status == 1 ] ; then - echo "Done" - exit 0 - else - attempts=$[ $attempts - 1 ] - - if [ $attempts == 0 ] ; then - break - fi - - echo "Sleeping as processes not stopped" - sleep 2 - - fi -done - -# If we haven't been able to kill the processes then -# forcibly do it -if [ ! $status == 1 ] ; then - forceStopRun - sleep 1 - status=`ps $PIDS |wc -l` - - if [ $status == 1 ] ; then - echo "Done" - else - echo "Stop failed" - exit 1 - fi -else - echo "Done" -fi diff --git a/java/perftests/bin/processing/process.sh b/java/perftests/bin/processing/process.sh deleted file mode 100755 index 6dbf9b9ea8..0000000000 --- a/java/perftests/bin/processing/process.sh +++ /dev/null @@ -1,388 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - - -usage() -{ - echo "usage: process.sh " - echo "These parameters are used to title and name the graphs:" - echo 'Title = $ $ : $% volume' - echo 'File = $-$-$-$' - exit 1 -} - -processCMSGCFile() -{ - # Extract all the ParNew timings: Ignore any lines corrupted by concurrent CMS logging. - grep -v CMS gc.log|grep ParNew > $work/parnew.gc.log - - # Calculate Full GC Data - grep failure -B 1 gc.log | sed -e :a -e '$!N;s/\n.(concurrent/ /;ta' -e 'P;D' |grep -v ^\- > $work/full.gc.log - cut -d ':' -f 1 $work/full.gc.log > $work/full.time.log - sed -e 's/failure)/\#/g' $work/full.gc.log |cut -d '#' -f 2- |awk '{print $3}' > $work/full.dur.log - - # Move to the working directory to further process the gathered data. - pushd $work &> /dev/null - - # Extract the Heap and timing data. - #8 is the full gc data - #5 is the paranew gc data - cat parnew.gc.log | awk '{print $8}' | cut -d '(' -f 2 | cut -d 'K' -f 1 > HEAP_MAX.log - cat parnew.gc.log | awk '{print $8}' | cut -d 'K' -f 1 > HEAP_PRE_GC.log - cat parnew.gc.log | awk '{print $8}' | cut -d 'K' -f 2 | cut -d '>' -f 2 > HEAP_POST_GC.log - cat parnew.gc.log | awk '{print $1}' | cut -d ':' -f 1 > GC_TIME.log - - calculateStats "MEM_PRE" HEAP_PRE_GC.log - calculateStats "MEM_POST" HEAP_POST_GC.log - calculateStatsBC "GC_FREQ" GC_TIME.log - - - # Calculate ParNew GC Cumulative total - cat parnew.gc.log |awk '{sum=sum+$6; print $6, sum}' > GC_DUR.log - # Give a count of GC occurances - parNewGCCount=`wc -l GC_DUR.log| awk '{print $1}'` - - # Create the Heap data file - paste GC_TIME.log HEAP_POST_GC.log HEAP_PRE_GC.log HEAP_MAX.log > GC.Heap.data - # Create ParNew GC Duration data file for graphing - paste GC_TIME.log GC_DUR.log > GC.Dur.data - # Create Full GC occurance log file for plotting - paste full.time.log full.dur.log > GC.Full.data - - # Calculate All GC Timing and give a count of their occurance - awk '{print $1}' GC_DUR.log > gc_dur.log - paste GC_TIME.log gc_dur.log > gc_all.log - cat GC.Full.data >> gc_all.log - sort -n gc_all.log | awk '{sum=sum+$2;print $1 , sum}' > GC.Dur.All.data - fullGCCount=`wc -l GC.Full.data| awk '{print $1}'` -} - -processG1GCFile() -{ - cat gc.log | grep -e \^[0-9]*.[0-9]*\: -e \^.*[0-9]*.\-\> > $work/g1.gc.log - - # Move to the working directory to further process the gathered data. - pushd $work &> /dev/null - - # Calculate Full GC Data - cat g1.gc.log | sed -e :a -e '$!N;s/\n.\ *\[/ \[/;ta' -e 'P;D' > full.gc.log - grep Full full.gc.log |awk '{print $1}'| tr ':' ' ' > full.time.log - grep Full full.gc.log |awk '{print $5}' > full.dur.log - fullGCCount=`wc -l GC.Full.data| awk '{print $1}'` - - # Create Full GC occurance log file for plotting - paste full.time.log full.dur.log > GC.Full.data - - # Extract the Heap and timing data. - # Create G1 Young Duration data file for graphing - grep "(young)," full.gc.log |awk '{print $1}' | tr ':' ' ' > GC_TIME.log - grep "(young)," full.gc.log |awk '{print $5}' > young.gc.time.log - # Calculate G1 young Cumulative total - cat young.gc.time.log |awk '{sum=sum+$1; print $1, sum}' > GC_DUR.log - # Create G1 Young Duration data file for graphing - paste GC_TIME.log GC_DUR.log > GC.Dur.data - - # Give a count of GC occurances - youngGCCount=`wc -l GC_DUR.log| awk '{print $1}'` - - # - # If we have no GCs then something is wrong - if [ $youngGCCount == 0 ] ; then - echo "Error no YoungGC log entries to proceses" - return - fi - - # Gather the Heap Size data - - grep "(young)," full.gc.log | awk '{print $8}' | cut -d '(' -f 2 | cut -d ')' -f 1 > HEAP_MAX.Sized.log - grep "(young)," full.gc.log | awk '{print $8}' | cut -d '-' -f 1 > HEAP_PRE_GC.Sized.log - grep "(young)," full.gc.log | awk '{print $8}' | cut -d '>' -f 2| cut -d '(' -f 1 > HEAP_POST_GC.Sized.log - - normaliseSizeMemFile HEAP_MAX.Sized.log HEAP_MAX.log - normaliseSizeMemFile HEAP_PRE_GC.Sized.log HEAP_PRE_GC.log - normaliseSizeMemFile HEAP_POST_GC.Sized.log HEAP_POST_GC.log - - - # Create the Heap data file - paste GC_TIME.log HEAP_POST_GC.log HEAP_PRE_GC.log HEAP_MAX.log > GC.Heap.data - - # Calculate All GC Timing and give a count of their occurance - awk '{print $1}' GC_DUR.log > gc_dur.log - paste GC_TIME.log gc_dur.log > gc_all.log - cat GC.Full.data >> gc_all.log - sort -n gc_all.log | awk '{sum=sum+$2;print $1 , sum}' > GC.Dur.All.data - -} - -calculateStatsBC() -{ -label=$1 -statFile=$2 -# Calculate GC Frequencies - prevFreq=0 - countFreq=0 - gcFreqTotal=0 - minFreq=99999999 - maxFreq=0 - - echo "" > GC_FREQ.log - for time in `cat $statFile` ; do - if [ $prevFreq == 0 ] ; then - prevFreq=$time - else - freq=`echo $time - $prevFreq | bc` - echo $freq >> GC_FREQ.log - - prevFreq=$time - gcFreqTotal=`echo $gcFreqTotal + $freq | bc` - countFreq=$[$countFreq + 1] - - if [ `echo "$freq > $maxFreq " | bc` == 1 ] ; then - maxFreq=$freq - fi - - if [ `echo "$freq < $minFreq " | bc ` == 1 ] ; then - minFreq=$freq - fi - fi - done - - # Extract Min/Max/Avg - echo "#type:min/max/avg" > $statFile.stats - - if [ $countFreq != 0 ] ; then - avgFreq=`echo $gcFreqTotal / $countFreq | bc -l 2&> /dev/null` - echo "$label:$minFreq/$maxFreq/$avgFreq" >> $statFile.stats - echo "Done GC Freq Stat generation for $statFile" - else - echo "$label:-/-/-" >> $statFile.stats - echo "Unable to calculate GC Freq stats as no freq entries found." - fi -} - -calculateStats() -{ -label=$1 -statFile=$2 - -# Calculate GC Frequencies - count=0 - gcTotal=0 - min=0 - max=0 - - - for item in `cat $statFile` ; do - if [ $min == 0 ] ; then - min=$item - fi - - gcTotal=$[$gcTotal + $item] - count=$[$count + 1] - - if [ $item -gt $max ] ; then - max=$item - fi - - if [ $item -lt $min ] ; then - min=$item - fi - done - - # Extract Min/Max/Avg - echo "#type:min/max/avg" > $statFile.stats - - if [ $count != 0 ] ; then - avg=`echo $gcTotal / $count | bc -l` - - echo "$label:$min/$max/$avg" >> $statFile.stats - echo "Done $label Stat generation for $statFile" - else - echo "$label:-/-/-" >> $statFile.stats - echo "Unable to calculate $label stats as no entries found." - fi - -} - -# -# Take an input file ($1) of lines -# -# and output file $2 of in whole M -# -normaliseSizeMemFile() -{ -rm -f $2 -for i in `cat $1` ; do - if [[ `echo $i | grep -c "K" ` == 1 ]] ; then - kb=`echo $i|cut -d 'K' -f 1` - echo $[ $kb / 1024 ] >> $2 - else - echo $i|cut -d 'M' -f 1 >> $2 - fi -done - - -} - - -# Parse command line -# TODO more advanced processing -# Check we have enough parameters -if [ $# != 4 ] ; then - # Take one arg to be a graph data file. - if [ $# == 1 ] ; then - textArray[0]="" # hold text - length=0 - # read whole file in loop - while read line - do - textArray[$length]=$line # store line - length=$(expr $length + 1) # increase length by 1 - done < $1 - - if [ $length != 2 ] ; then - usage - fi - - #Get Title and file name - title=${textArray[0]} - file=${textArray[1]} - - pushd `dirname $1` - - else - usage - fi -else - version=$1 - type=$2 - volume=$3 - brokerState=$4 - - - # Configure Graph Title and image file names - title="$version $type : $volume% volume" - file="$version-$brokerState-$type-$volume" -fi - -work=work - -mkdir -p $work - -echo -n "Processing GC Usage Data : " -ParNew=`grep -c ParNew gc.log` - -if [ $ParNew != 0 ] ; then - echo "CMS log file" - processCMSGCFile - PLOT="\"GC.Dur.data\" with lines axis x1y1 ti \"ParNew GC Time ($parNewGCCount)\", " -else - echo "G1 log file" - processG1GCFile - PLOT="\"GC.Dur.data\" with lines axis x1y1 ti \"G1 Young Time ($youngGCCount)\", " -fi - -# -# Colate Stats -# -echo -n "Colating stat Data : " -echo "#type:min/max/avg" > $file.statistics.txt -find .. -name "*stats" -exec grep -v "type" {} >> $file.statistics.txt \; - - -# Prepare the plot command -# If a Full GC occured during this test then plot those -if [[ $fullGCCount > 0 ]] ; then -PLOT=$PLOT"\"GC.Dur.data\" using 1:3 with lines axis x1y2 ti \"Cumulative Total Time(ParNew)\", \ - \"GC.Dur.All.data\" with lines axis x1y2 ti \"Cumulative Total Time(All)\", \ - \"GC.Full.data\" with points ti \"Full GCs Time ($fullGCCount)\" " -else -PLOT=$PLOT"\"GC.Dur.data\" using 1:3 with lines axis x1y2 ti \"Cumulative Total Time(ParNew)\", \ - \"GC.Dur.All.data\" with lines axis x1y2 ti \"Cumulative Total Time(All)\" " -fi - -if [ $ParNew != 0 ] ; then - gcs=$parNewGCCount -else - gcs=$youngGCCount -fi - -# Call out to gnuplot to generate graphs -# Only do this if we have data -if [ $gcs != 0 ] ; then - # Generate the Heap Graph and the GC Duration Graph - gnuplot << EOGNUPLOT -set xlabel "Run Time(s)" - -set title "$title : Heap Size" -set term png -set output "$file-Heap.png" -set ylabel "MB" +0.5,0 -plot "GC.Heap.data" using 1:2 with lines axis x1y1 ti "GC Size Post",\ - "GC.Heap.data" using 1:4 with lines axis x1y1 ti "GC Size Max ", \ - "GC.Heap.data" using 1:3 with lines axis x1y1 ti "GC Size Pre " - -set y2tics nomirror -set ytics nomirror -set key top left -set title "$title GC Time" -set ylabel "Time(s)" +0.5,0 -set y2label "Total Time(s)" -set output "$file-GCDuration.png" -plot $PLOT -EOGNUPLOT - -else - echo "Warning: No GC Data to graph" -fi - -# pop back to further process for CPU usage -popd &> /dev/null - -echo "Processing CPU Usage Data" - -# CPU.data is just TimeStamp + %age -cat broker_cpu.log |awk '{print $1 "T" $2 " " $3}' > $work/CPU.data - -# Move to work directory to run gnuplot -pushd $work &> /dev/null - -# Call out to gnuplot to generate graphs -# Generate the Heap Graph and the GC Duration Graph -gnuplot << EOGNUPLOT -set term png -set title "$title : CPU" -set output "$file-CPU.png" -unset key -set xlabel "Real Time(h:m)" -set ylabel "Usage(%)" - -set xdata time -set timefmt "%Y-%m-%dT%H:%M:%S" -set format x "%H:%M" -plot "CPU.data" using 1:2 with lines -EOGNUPLOT - -#" -# Pop out of $work directory. -# -popd &> /dev/null \ No newline at end of file diff --git a/java/perftests/bin/processing/processAll.sh b/java/perftests/bin/processing/processAll.sh deleted file mode 100755 index 895d59c316..0000000000 --- a/java/perftests/bin/processing/processAll.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - - -usage() -{ - echo "processResults.sh " - exit 1 -} - -root=`pwd` -echo $root -graphFile=graph.data - -if [ $# != 1 ] ; then - usage -fi - -mkdir -p results - -for file in `find $1 -name $graphFile` ; do - - dir=`dirname $file` - - echo Processing : $dir - pushd $dir &> /dev/null - - $root/process.sh $graphFile - - echo Copying Images - cp work/*png $root/results/ - - echo Copying Stats - cp work/*.statistics.txt $root/results/ - - - popd &> /dev/null - echo Done -done \ No newline at end of file diff --git a/java/perftests/bin/processing/processTests.py b/java/perftests/bin/processing/processTests.py deleted file mode 100755 index 8db44eda79..0000000000 --- a/java/perftests/bin/processing/processTests.py +++ /dev/null @@ -1,850 +0,0 @@ -#!/usr/bin/env python -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import re -import datetime -import sys -import string -from optparse import OptionParser -from datetime import datetime, timedelta -import shutil - - -def showUsage(): - log("./processTests.py [-b|--broker-log-dir]

[-t|--test-dir] ") - - -ACCESS="Access" -MODIFY="Modify" - -BROKER_LOG="broker.log" -BROKER_PID="broker.pid" -BROKER_CPU="broker_cpu.log" -BROKER_CPU_DATED="broker_cpu.log.dated" -BROKER_STATS="broker.stats" -BROKER_GC="gc.log" - -GRAPH_DATA="graph.data" - -_verbose = False -_debug = False -_brokerLogs = "" - -def exitError(message): - log(message) - sys.exit(1) - -def main(): - global _log, _verbose, _debug, _brokerLogs - - # Load the - parser = OptionParser() - - parser.add_option("-v", "--verbose", dest="verbose", - action="store_true", default=False, help="enable verbose output") - - parser.add_option("-d", "--debug", dest="debug", - action="store_true", default=False, help="enable debug output") - - parser.add_option("-b", "--broker-log-dir", dest="brokerLogs", - action="store", default=True, help="Broker Logs") - - parser.add_option("-t", "--test-dir", dest="testDir", - action="store", default="", help="Test Results") - - - (options, args) = parser.parse_args() - - _verbose = options.verbose - _debug = options.debug - testDir = options.testDir - _brokerLogs = options.brokerLogs - - if testDir == "" or _brokerLogs == "" : - log("Broker Log Dir and Test Dir are both requried.") - showUsage() - - if not os.path.exists(testDir): - exitError("Test directory does not exist:" + testDir) - - if not os.path.exists(_brokerLogs): - exitError("Broker log directory does not exist:" + _brokerLogs) - - - # Standardize the format of the broker logs - preProcessBrokerLogs(_brokerLogs) - - # Get list of test results from test_dir - processTestResults(testDir) - -# -# Process the log files we know of -# -def preProcessBrokerLogs(resultDir): - - print "Pre Processing Broker Logs" - # Pre-Process GC - no pre processing required - - # Process Log4j - no processing required as file is already time stamped. - - # Pre-Process broker_cpu - processCPUUsage(resultDir) - -# -# Process the broker CPU log file and create an output file of format -# -# -# -def processCPUUsage(resultDir): - logfile=resultDir+os.sep+BROKER_CPU - datedFile=resultDir+os.sep+BROKER_CPU_DATED - - start = extractTime(ACCESS, logfile+".stat") - - pid = getPID(BROKER_PID) - - topRate = getFirstLine(_brokerLogs+os.sep+"top.rate") - - # - # Calulate addition required per process line output - # - if topRate.find(".") == -1: - seconds = topRate - millis = 0 - else: - split = topRate.split('.') - seconds = split[0] - # Convert - millis = float("0."+split[1]) * 1000 - - offset = timedelta(seconds=int(seconds),milliseconds=int(millis)) - - # - # Process the CPU log file and make a file of format: - # datetime - # - # Open log CPU file for reading - logfile = open(logfile, "r") - - # Open the output file, erasing any existing version - # Keep track of the min/max sum and entries,. - minCPU=float(sys.maxint) - maxCPU=0.0 - minMem=float(sys.maxint) - maxMem=0.0 - - entries=0 - sumCPU=0.0 - sumMem=0.0 - - output= open(datedFile, "w") - for line in logfile: - # - # Data format - # 0 1 2 3 4 5 6 7 8 9 10 11 - # PID USER PR NI %CPU TIME+ %MEM VIRT RES SHR S COMMAND - # PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND - # - # %CPU and %MEM are vary, probably based on os/version of top. - # lets auto-detect where it is. - # - # Index is 0 based for array usage. - index = 0 - if line.find("PID") != -1: - for key in line.split(" "): - strippedKey = key.lstrip() - if len(strippedKey) > 0: - # Record the key index - if (strippedKey == "%CPU"): - cpuIndex=index - if (strippedKey == "%MEM"): - memIndex=index - # Increase count for next key - index = index + 1 - - - # Find lines that contain our broker process - if line.find("QPBRKR") != -1: - - # Split line on whitespace - data = line.split() - - #Write out the date time (ISO-8601 format) - output.write(str(start)) - # Output the %CPU value - output.write(" "+str(data[cpuIndex])) - # Output the %MEM value - output.write(" "+str(data[memIndex])) - output.write('\n') - - # Add the offset based on the logging rate - start = start + offset - - # Record entires - entries = entries + 1 - - # Record Metrics - # Record CPU data - cpu = float(data[cpuIndex]) - if (cpu < minCPU): - minCPU = cpu - - if (cpu > maxCPU): - maxCPU = cpu - - sumCPU = sumCPU + cpu - - # Record Mem data - mem = float(data[memIndex]) - - if (mem < minMem): - minMem = mem - - if (mem > maxMem): - maxMem = mem - - sumMem = sumMem + mem - - - #end for - - # Close the files - logfile.close - output.close - - # Output stats file - statFile=resultDir+os.sep+BROKER_CPU+".stats" - output= open(statFile, "w") - output.write("#type:min/max/avg") - output.write('\n') - output.write("CPU:"+str(minCPU)+"/"+str(maxCPU)+"/"+str(float(sumCPU)/float(entries))) - output.write('\n') - output.write("MEM:"+str(minMem)+"/"+str(maxMem)+"/"+str(float(sumMem)/float(entries))) - output.write('\n') - output.close - - log("Pre Process of CPU Log file '"+BROKER_CPU+"' complete") - - -# -# Give an known process type get the recorded PID. -# -def getPID(process): - return getFirstLine(_brokerLogs+os.sep+process) - -# -# Get the first line of the file without EOL chars. -# NOTE: this will load the entire file into memory to do it. -# -def getFirstLine(fileName): - f = open(fileName,"r") - line = f.read().splitlines()[0] - f.close - return line - - -# -# Walk the directory given and process all csv test results -# -def processTestResults(resultDir): - for root, dirs, files in os.walk(resultDir, topdown=False): - if len(files) == 0: - exitError("Test result directory is empty:" + resultDir) - for file in files: - if file.endswith(".csv"): - processTestResult(root , file) - -def processTestResult(root, resultFile): - # Open stat file and extract test times, we determine: - # -start time based on the 'Access' value - # -end time based on the 'Modify' value 'Change' would also work - - statFile=root+os.sep+resultFile+".stat" - - if not os.path.exists(statFile): - log("Unable to process : Unable to open stat file:" + statFile) - return - - - createResultSetPackage(root, resultFile) - - -def extractTime(field, statFile): - stats = open(statFile, "r") - for line in stats: - if line.startswith(field): - if line.find("(") == -1: - dt = lineToDate(" ".join(line.split()[1:])) - - # - # TODO We need to handle time time zone issues as I'm sure we will have issues with the - # log4j matching. - - stats.close - return dt - -# -# Given a text line in ISO format convert it to a date object -# -def lineToDate(line): - #2009-06-22 17:04:44,320 - #2009-06-22 17:04:44.320 - pattern = re.compile(r'(?P^[0-9][0-9][0-9][0-9])-(?P[0-9][0-9])-(?P[0-9][0-9]) (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])') - - - m = pattern.match(line) - if m: - year = int(m.group('year')) - month = int(m.group('month')) - day = int(m.group('day')) - hour = int(m.group('hour')) - minute = int(m.group('minute')) - seconds = int(m.group('seconds')) - - pattern = re.compile(r'(?P^[0-9][0-9][0-9][0-9])-(?P[0-9][0-9])-(?P[0-9][0-9]) (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])[.|,](?P[0-9]+)') - m = pattern.match(line) - micro = None - if m: - micro = m.group('micro') - - if micro == None: - micro = 0 - - # Correct issue where micros are actually nanos - if int(micro) > 999999: - micro = int(micro) / 1000 - - return datetime(year,month,day,hour,minute,seconds,int(micro)) - else: - # Error we shouldn't get here - return null - -def createResultSetPackage(root, resultFile): - # Get the Name of the test to make a directory with said name - testName = resultFile.split(".csv")[0] - resultDir = root+ os.sep + testName - - log("Processing Result set for:"+ testName) - - mkdir(resultDir) - - # Move result file to new directory - shutil.move(root + os.sep + resultFile, resultDir) - - # Move stat file to new directory - shutil.move(root + os.sep + resultFile + ".stat", resultDir) - - statFile=resultDir + os.sep + resultFile + ".stat" - - # - # Get start and end time for test run - # - start = extractTime(ACCESS, statFile) - end = extractTime(MODIFY, statFile) - - sliceBrokerLogs(resultDir, start, end) - createGraphData(resultDir, testName) - createTestStatData(resultDir, testName) - - log("Created Result Package for:"+ testName) - -def sliceBrokerLogs(resultDir, start, end): - sliceCPULog(resultDir, start, end) - sliceLog4j(resultDir, start, end) - sliceGCLog(resultDir, start, end) - - -def sliceCPULog(resultDir, start, end): - global _brokerLogs - logfilePath=_brokerLogs+os.sep+BROKER_CPU_DATED - cpuSliceFile=resultDir+os.sep+BROKER_CPU - - # Process the CPU log file and make a file of format: - # datetime - # - # Open log CPU file for reading - logFile = open(logfilePath, "r") - - # Open the output file, erasing any existing version - # Keep track of the min/max sum and entries,. - minCPU=float(sys.maxint) - maxCPU=0.0 - minMem=float(sys.maxint) - maxMem=0.0 - - entries=0 - sumCPU=0.0 - sumMem=0.0 - - # - # Create outputfile - # - cpuslice = open(cpuSliceFile,"w") - for line in logFile: - data = line.split() - # - # //fixme remove tz addition. - # - lineTime = lineToDate(" ".join(data[0:2])+" +0000") - - if lineTime > start: - if lineTime < end: - # Write the data though to the new file - cpuslice.writelines(line) - - # Perform stat processing for the min/max/avg - - data = line.split() - - # - # Data format is - #