From 5749696efa8958eb3a6d0ad0c166c48d2893c429 Mon Sep 17 00:00:00 2001 From: Robert Greig Date: Tue, 8 May 2007 11:21:35 +0000 Subject: Added XML logging of test results. git-svn-id: https://svn.apache.org/repos/asf/incubator/qpid/branches/M2@536163 13f79535-47bb-0310-9956-ffa450edef68 --- .../qpid/interop/coordinator/Coordinator.java | 89 ++++- .../interop/coordinator/InvitingTestDecorator.java | 10 + .../qpid/interop/coordinator/XMLTestListener.java | 381 +++++++++++++++++++++ 3 files changed, 475 insertions(+), 5 deletions(-) create mode 100644 java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java (limited to 'java/integrationtests/src') diff --git a/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java b/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java index 9a1236132e..de5faeac0f 100644 --- a/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java +++ b/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/Coordinator.java @@ -20,6 +20,7 @@ */ package org.apache.qpid.interop.coordinator; +import java.io.*; import java.util.*; import java.util.concurrent.LinkedBlockingQueue; @@ -39,8 +40,10 @@ import org.apache.qpid.util.CommandLineParser; import org.apache.qpid.util.ConversationFactory; import org.apache.qpid.util.PrettyPrintingUtils; -import uk.co.thebadgerset.junit.extensions.TestRunnerImprovedErrorHandling; +import uk.co.thebadgerset.junit.extensions.TKTestResult; +import uk.co.thebadgerset.junit.extensions.TKTestRunner; import uk.co.thebadgerset.junit.extensions.WrappedSuiteTestDecorator; +import uk.co.thebadgerset.junit.extensions.util.TestContextProperties; /** *

Implements the coordinator client described in the interop testing specification @@ -55,7 +58,7 @@ import uk.co.thebadgerset.junit.extensions.WrappedSuiteTestDecorator; * Terminate the interop testing framework. * */ -public class Coordinator extends TestRunnerImprovedErrorHandling +public class Coordinator extends TKTestRunner { private static final Logger log = Logger.getLogger(Coordinator.class); @@ -76,6 +79,15 @@ public class Coordinator extends TestRunnerImprovedErrorHandling /** Holds the connection that the coordinating messages are sent over. */ private Connection connection; + /** + * Holds the name of the class of the test currently being run. Ideally passed into the {@link #createTestResult} + * method, but as the signature is already fixed for this, the current value gets pushed here as a member variable. + */ + private String currentTestClassName; + + /** Holds the path of the directory to output test results too, if one is defined. */ + private static String reportDir; + /** * Creates an interop test coordinator on the specified broker and virtual host. * @@ -114,12 +126,14 @@ public class Coordinator extends TestRunnerImprovedErrorHandling new String[][] { { "b", "The broker URL.", "broker", "false" }, - { "h", "The virtual host to use.", "virtual host", "false" } + { "h", "The virtual host to use.", "virtual host", "false" }, + { "o", "The name of the directory to output test timings to.", "dir", "false" } })); // Extract the command line options. String brokerUrl = options.getProperty("b"); String virtualHost = options.getProperty("h"); + reportDir = options.getProperty("o"); // Scan for available test cases using a classpath scanner. Collection> testCaseClasses = @@ -183,7 +197,8 @@ public class Coordinator extends TestRunnerImprovedErrorHandling */ public TestResult start(String[] testClassNames) throws Exception { - log.debug("public TestResult start(String testClassName): called"); + log.debug("public TestResult start(String[] testClassNames = " + PrettyPrintingUtils.printArray(testClassNames) + + ": called"); // Connect to the broker. connection = TestClient.createConnection(DEFAULT_CONNECTION_PROPS_RESOURCE, brokerUrl, virtualHost); @@ -214,6 +229,9 @@ public class Coordinator extends TestRunnerImprovedErrorHandling for (String testClassName : testClassNames) { + // Record the current test class, so that the test results can be output to a file incorporating this name. + this.currentTestClassName = testClassName; + result = super.start(new String[] { testClassName }); } @@ -300,6 +318,67 @@ public class Coordinator extends TestRunnerImprovedErrorHandling // Wrap the tests in an inviting test decorator, to perform the invite/test cycle. targetTest = new InvitingTestDecorator(targetTest, enlistedClients, conversationFactory, connection); - return super.doRun(targetTest, wait); + TestSuite suite = new TestSuite(); + suite.addTest(targetTest); + + // Wrap the tests in a scaled test decorator to them them as a 'batch' in one thread. + // targetTest = new ScaledTestDecorator(targetTest, new int[] { 1 }); + + return super.doRun(suite, wait); + } + + /** + * Creates the TestResult object to be used for test runs. + * + * @return An instance of the test result object. + */ + protected TestResult createTestResult() + { + log.debug("protected TestResult createTestResult(): called"); + + TKTestResult result = new TKTestResult(fPrinter.getWriter(), delay, verbose, testCaseName); + + // Check if a directory to output reports to has been specified and attach test listeners if so. + if (reportDir != null) + { + // Create the report directory if it does not already exist. + File reportDirFile = new File(reportDir); + + if (!reportDirFile.exists()) + { + reportDirFile.mkdir(); + } + + // Create the timings file (make the name of this configurable as a command line parameter). + Writer timingsWriter = null; + + try + { + File timingsFile = new File(reportDirFile, "TEST." + currentTestClassName + ".xml"); + timingsWriter = new BufferedWriter(new FileWriter(timingsFile), 20000); + } + catch (IOException e) + { + throw new RuntimeException("Unable to create the log file to write test results to: " + e, e); + } + + // Set up a CSV results listener to output the timings to the results file. + XMLTestListener listener = new XMLTestListener(timingsWriter, currentTestClassName); + result.addListener(listener); + result.addTKTestListener(listener); + + // Register the results listeners shutdown hook to flush its data if the test framework is shutdown + // prematurely. + // registerShutdownHook(listener); + + // Record the start time of the batch. + // result.notifyStartBatch(); + + // At this point in time the test class has been instantiated, giving it an opportunity to read its parameters. + // Inform any test listers of the test properties. + result.notifyTestProperties(TestContextProperties.getAccessedProps()); + } + + return result; } } diff --git a/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java b/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java index afdad70001..858ed1a589 100644 --- a/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java +++ b/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/InvitingTestDecorator.java @@ -162,6 +162,16 @@ public class InvitingTestDecorator extends WrappedSuiteTestDecorator } } + /** + * Prints a string summarizing this test decorator, mainly for debugging purposes. + * + * @return String representation for debugging purposes. + */ + public String toString() + { + return "InvitingTestDecorator: [ testSuite = " + testSuite + " ]"; + } + /** * Produces all pairs of combinations of elements from two sets. The ordering of the elements in the pair is * important, that is the pair is distinct from ; both pairs are generated. For any element, i, in diff --git a/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java b/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java new file mode 100644 index 0000000000..6c7a2a5d52 --- /dev/null +++ b/java/integrationtests/src/main/java/org/apache/qpid/interop/coordinator/XMLTestListener.java @@ -0,0 +1,381 @@ +package org.apache.qpid.interop.coordinator; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.util.*; + +import junit.framework.AssertionFailedError; +import junit.framework.Test; +import junit.framework.TestCase; + +import org.apache.log4j.Logger; + +import uk.co.thebadgerset.junit.extensions.listeners.TKTestListener; + +/** + * Listens for test results for a named test and outputs these in the standard JUnit XML format to the specified + * writer. + * + *

The API for this listener accepts notifications about different aspects of a tests results through different + * methods, so some assumption needs to be made as to which test result a notification refers to. For example + * {@link #startTest} will be called, then possibly {@link #timing} will be called, even though the test instance is + * passed in both cases, it is not enough to distinguish a particular run of the test, as the test case instance may + * be being shared between multiple threads, or being run a repeated number of times, and can therfore be re-used + * between calls. The listeners make the assumption that, for every test, a unique thread will call {@link #startTest} + * and {@link #endTest} to delimit each test. All calls to set test parameters, timings, state and so on, will occur + * between the start and end and will be given with the same thread id as the start and end, so the thread id provides + * a unqiue value to identify a particular test run against. + * + *

+ *
CRC Card
Responsibilities Collaborations + *
+ * + * @todo Merge this class with CSV test listener, making the collection of results common to both, and only factoring + * out the results printing code into sub-classes. Provide a simple XML results formatter with the same format as + * the ant XML formatter, and a more structured one for outputing results with timings and summaries from + * performance tests. + */ +public class XMLTestListener implements TKTestListener +{ + /** Used for debugging. */ + private static final Logger log = Logger.getLogger(XMLTestListener.class); + + /** The results file writer. */ + protected Writer writer; + + /** Holds the results for individual tests. */ + // protected Map results = new LinkedHashMap(); + // protected List results = new ArrayList(); + + /** + * Map for holding results on a per thread basis as they come in. A ThreadLocal is not used as sometimes an + * explicit thread id must be used, where notifications come from different threads than the ones that called + * the test method. + */ + Map threadLocalResults = Collections.synchronizedMap(new LinkedHashMap()); + + /** + * Holds results for tests that have ended. Transferring these results here from the per-thread results map, means + * that the thread id is freed for the thread to generate more results. + */ + List results = new ArrayList(); + + /** Holds the overall error count. */ + protected int errors = 0; + + /** Holds the overall failure count. */ + protected int failures = 0; + + /** Holds the overall tests run count. */ + protected int runs = 0; + + /** Holds the name of the class that tests are being run for. */ + String testClassName; + + /** + * Creates a new XML results output listener that writes to the specified location. + * + * @param writer The location to write results to. + */ + public XMLTestListener(Writer writer, String testClassName) + { + log.debug("public XMLTestListener(Writer writer, String testClassName = " + testClassName + "): called"); + + this.writer = writer; + this.testClassName = testClassName; + } + + /** + * Resets the test results to the default state of time zero, memory usage zero, parameter zero, test passed. + * + * @param test The test to resest any results for. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void reset(Test test, Long threadId) + { + log.debug("public void reset(Test test = " + test + ", Long threadId = " + threadId + "): called"); + + XMLTestListener.Result r = + (threadId == null) ? threadLocalResults.get(Thread.currentThread().getId()) : threadLocalResults.get(threadId); + + r.error = null; + r.failure = null; + + } + + /** + * A test started. + */ + public void startTest(Test test) + { + log.debug("public void startTest(Test test = " + test + "): called"); + + Result newResult = new Result(test.getClass().getName(), ((TestCase) test).getName()); + + // Initialize the thread local test results. + threadLocalResults.put(Thread.currentThread().getId(), newResult); + runs++; + } + + /** + * Should be called every time a test completes with the run time of that test. + * + * @param test The name of the test. + * @param nanos The run time of the test in nanoseconds. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void timing(Test test, long nanos, Long threadId) + { } + + /** + * Should be called every time a test completed with the amount of memory used before and after the test was run. + * + * @param test The test which memory was measured for. + * @param memStart The total JVM memory used before the test was run. + * @param memEnd The total JVM memory used after the test was run. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void memoryUsed(Test test, long memStart, long memEnd, Long threadId) + { } + + /** + * Should be called every time a parameterized test completed with the int value of its test parameter. + * + * @param test The test which memory was measured for. + * @param parameter The int parameter value. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void parameterValue(Test test, int parameter, Long threadId) + { } + + /** + * Should be called every time a test completes with the current number of test threads running. + * + * @param test The test for which the measurement is being generated. + * @param threads The number of tests being run concurrently. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void concurrencyLevel(Test test, int threads, Long threadId) + { } + + /** + * Notifies listeners of the tests read/set properties. + * + * @param properties The tests read/set properties. + */ + public void properties(Properties properties) + { } + + /** + * A test ended. + */ + public void endTest(Test test) + { + log.debug("public void endTest(Test test = " + test + "): called"); + + // Move complete test results into the completed tests list. + Result r = threadLocalResults.get(Thread.currentThread().getId()); + results.add(r); + + // Clear all the test results for the thread. + threadLocalResults.remove(Thread.currentThread().getId()); + } + + /** + * Called when a test completes. Success, failure and errors. This method should be used when registering an + * end test from a different thread than the one that started the test. + * + * @param test The test which completed. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void endTest(Test test, Long threadId) + { + log.debug("public void endTest(Test test = " + test + ", Long threadId = " + threadId + "): called"); + + // Move complete test results into the completed tests list. + Result r = + (threadId == null) ? threadLocalResults.get(Thread.currentThread().getId()) : threadLocalResults.get(threadId); + results.add(r); + + // Clear all the test results for the thread. + threadLocalResults.remove(Thread.currentThread().getId()); + } + + /** + * An error occurred. + */ + public void addError(Test test, Throwable t) + { + log.debug("public void addError(Test test = " + test + ", Throwable t = " + t + "): called"); + + Result r = threadLocalResults.get(Thread.currentThread().getId()); + r.error = t; + errors++; + } + + /** + * A failure occurred. + */ + public void addFailure(Test test, AssertionFailedError t) + { + log.debug("public void addFailure(Test test = " + test + ", AssertionFailedError t = " + t + "): called"); + + Result r = threadLocalResults.get(Thread.currentThread().getId()); + r.failure = t; + failures++; + } + + /** + * Called when a test completes to mark it as a test fail. This method should be used when registering a + * failure from a different thread than the one that started the test. + * + * @param test The test which failed. + * @param e The assertion that failed the test. + * @param threadId Optional thread id if not calling from thread that started the test method. May be null. + */ + public void addFailure(Test test, AssertionFailedError e, Long threadId) + { + log.debug("public void addFailure(Test test, AssertionFailedError e, Long threadId): called"); + + Result r = + (threadId == null) ? threadLocalResults.get(Thread.currentThread().getId()) : threadLocalResults.get(threadId); + r.failure = e; + failures++; + } + + /** + * Notifies listeners of the start of a complete run of tests. + */ + public void startBatch() + { + log.debug("public void startBatch(): called"); + + // Reset all results counts. + threadLocalResults = Collections.synchronizedMap(new HashMap()); + errors = 0; + failures = 0; + runs = 0; + + // Write out the file header. + try + { + writer.write("\n"); + } + catch (IOException e) + { + throw new RuntimeException("Unable to write the test results.", e); + } + } + + /** + * Notifies listeners of the end of a complete run of tests. + * + * @param parameters The optional test parameters to log out with the batch results. + */ + public void endBatch(Properties parameters) + { + log.debug("public void endBatch(Properties parameters = " + parameters + "): called"); + + // Write out the results. + try + { + // writer.write("\n"); + writer.write("\n"); + + for (Result result : results) + { + writer.write(" \n"); + + if (result.error != null) + { + writer.write(" "); + result.error.printStackTrace(new PrintWriter(writer)); + writer.write(" "); + } + else if (result.failure != null) + { + writer.write(" "); + result.failure.printStackTrace(new PrintWriter(writer)); + writer.write(" "); + } + + writer.write(" \n"); + } + + writer.write("\n"); + writer.flush(); + } + catch (IOException e) + { + throw new RuntimeException("Unable to write the test results.", e); + } + } + + /** + * Used to capture the results of a particular test run. + */ + protected static class Result + { + public Result(String testClass, String testName) + { + this.testClass = testClass; + this.testName = testName; + } + + public String testClass; + public String testName; + + /** Holds the exception that caused error in this test. */ + public Throwable error; + + /** Holds the assertion exception that caused failure in this test. */ + public AssertionFailedError failure; + + /** Holds the error count for this test. */ + // public int errors = 0; + + /** Holds the failure count for this tests. */ + // public int failures = 0; + + /** Holds the overall tests run count for this test. */ + // public int runs = 0; + + /*public boolean equals(Object o) + { + if (this == o) + { + return true; + } + + if (!(o instanceof Result)) + { + return false; + } + + final Result result = (Result) o; + + if ((testClass != null) ? (!testClass.equals(result.testClass)) : (result.testClass != null)) + { + return false; + } + + if ((testName != null) ? (!testName.equals(result.testName)) : (result.testName != null)) + { + return false; + } + + return true; + } + + public int hashCode() + { + int result; + result = ((testClass != null) ? testClass.hashCode() : 0); + result = (29 * result) + ((testName != null) ? testName.hashCode() : 0); + + return result; + }*/ + } +} -- cgit v1.2.1