QPID-5991: [Java System Tests] BDB HA System Test Improvements

* Change failover url so that algorithm for cases where we expect a connection to the broker checks each broker once
  per cycle rather than twice successively.  Doubled the number of cycles and reduced the connect delay so that
  test discover the master node more quickly.
* Use separate negative failover url when expectation is that group will refuse connection.
* Remove duplicated test
* In TwoNodeTest use REST API only.   JMXManagementTest now has sole responsibility for testing HA using the JMX bean.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1617895 13f79535-47bb-0310-9956-ffa450edef68

3 files changed, 38 insertions, 64 deletions

diff --git a/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/GroupCreator.java b/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/GroupCreator.java
index e78ef34759..f7dce4f3f5 100644
--- a/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/GroupCreator.java
+++ b/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/GroupCreator.java
@@ -75,9 +75,9 @@ public class GroupCreator
     private static final String MANY_BROKER_URL_FORMAT = "amqp://guest:guest@/%s?brokerlist='%s'&failover='roundrobin?cyclecount='%d''";
     private static final String BROKER_PORTION_FORMAT = "tcp://localhost:%d?connectdelay='%d',retries='%d'";
 
-    private static final int FAILOVER_CYCLECOUNT = 10;
-    private static final int FAILOVER_RETRIES = 1;
-    private static final int FAILOVER_CONNECTDELAY = 1000;
+    private static final int FAILOVER_CYCLECOUNT = 20;
+    private static final int FAILOVER_RETRIES = 0;
+    private static final int FAILOVER_CONNECTDELAY = 500;
 
     private static final String SINGLE_BROKER_URL_WITH_RETRY_FORMAT = "amqp://guest:guest@/%s?brokerlist='tcp://localhost:%d?connectdelay='%d',retries='%d''";
     private static final String SINGLE_BROKER_URL_WITHOUT_RETRY_FORMAT = "amqp://guest:guest@/%s?brokerlist='tcp://localhost:%d'";
diff --git a/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/JMXManagementTest.java b/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/JMXManagementTest.java
index c6f005c0e7..63de287be7 100644
--- a/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/JMXManagementTest.java
+++ b/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/JMXManagementTest.java
@@ -206,20 +206,6 @@ public class JMXManagementTest extends QpidBrokerTestCase
         }
     }
 
-    public void testSetDesignatedPrimary() throws Exception
-    {
-        int brokerPort = _clusterCreator.getBrokerPortNumbersForNodes().iterator().next();
-        final ManagedBDBHAMessageStore storeBean = getStoreBeanForNodeAtBrokerPort(brokerPort);
-        assertFalse("Unexpected designated primary before change", storeBean.getDesignatedPrimary());
-        storeBean.setDesignatedPrimary(true);
-        long limit = System.currentTimeMillis() + 5000;
-        while(!storeBean.getDesignatedPrimary() && System.currentTimeMillis() < limit)
-        {
-            Thread.sleep(100l);
-        }
-        assertTrue("Unexpected designated primary after change", storeBean.getDesignatedPrimary());
-    }
-
     public void testVirtualHostMbeanOnMasterTransfer() throws Exception
     {
         Connection connection = getConnection(_brokerFailoverUrl);
diff --git a/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/TwoNodeTest.java b/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/TwoNodeTest.java
index 0f8a1609de..248dbb4def 100644
--- a/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/TwoNodeTest.java
+++ b/qpid/java/bdbstore/systests/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/TwoNodeTest.java
@@ -20,27 +20,29 @@
 package org.apache.qpid.server.store.berkeleydb.replication;
 
 import java.io.File;
+import java.util.Collections;
+import java.util.Map;
 
 import javax.jms.Connection;
 import javax.jms.JMSException;
-import javax.management.ObjectName;
 
 import org.apache.qpid.jms.ConnectionURL;
-import org.apache.qpid.server.store.berkeleydb.jmx.ManagedBDBHAMessageStore;
-import org.apache.qpid.test.utils.JMXTestUtils;
+import org.apache.qpid.server.virtualhostnode.berkeleydb.BDBHAVirtualHostNode;
 import org.apache.qpid.test.utils.QpidBrokerTestCase;
 
 public class TwoNodeTest extends QpidBrokerTestCase
 {
     private static final String VIRTUAL_HOST = "test";
 
-    private static final String MANAGED_OBJECT_QUERY = "org.apache.qpid:type=BDBHAMessageStore,name=" + ObjectName.quote(VIRTUAL_HOST);
     private static final int NUMBER_OF_NODES = 2;
 
     private final GroupCreator _groupCreator = new GroupCreator(this, VIRTUAL_HOST, NUMBER_OF_NODES);
-    private final JMXTestUtils _jmxUtils = new JMXTestUtils(this);
 
-    private ConnectionURL _brokerFailoverUrl;
+    /** Used when expectation is client will not (re)-connect */
+    private ConnectionURL _positiveFailoverUrl;
+
+    /** Used when expectation is client will not (re)-connect */
+    private ConnectionURL _negativeFailoverUrl;
 
     @Override
     protected void setUp() throws Exception
@@ -54,19 +56,6 @@ public class TwoNodeTest extends QpidBrokerTestCase
     }
 
     @Override
-    protected void tearDown() throws Exception
-    {
-        try
-        {
-            _jmxUtils.close();
-        }
-        finally
-        {
-            super.tearDown();
-        }
-    }
-
-    @Override
     public void startBroker() throws Exception
     {
         // Don't start default broker provided by QBTC.
@@ -77,20 +66,21 @@ public class TwoNodeTest extends QpidBrokerTestCase
         setSystemProperty("java.util.logging.config.file", "etc" + File.separator + "log.properties");
         _groupCreator.configureClusterNodes();
         _groupCreator.setDesignatedPrimaryOnFirstBroker(designedPrimary);
-        _brokerFailoverUrl = _groupCreator.getConnectionUrlForAllClusterNodes();
+        _positiveFailoverUrl = _groupCreator.getConnectionUrlForAllClusterNodes();
+        _negativeFailoverUrl = _groupCreator.getConnectionUrlForAllClusterNodes(200, 0, 2);
         _groupCreator.startCluster();
     }
 
     public void testMasterDesignatedPrimaryCanBeRestartedWithoutReplica() throws Exception
     {
         startCluster(true);
-        final Connection initialConnection = getConnection(_brokerFailoverUrl);
+        final Connection initialConnection = getConnection(_positiveFailoverUrl);
         int masterPort = _groupCreator.getBrokerPortNumberFromConnection(initialConnection);
         assertProducingConsuming(initialConnection);
         initialConnection.close();
         _groupCreator.stopCluster();
         _groupCreator.startNode(masterPort);
-        final Connection secondConnection = getConnection(_brokerFailoverUrl);
+        final Connection secondConnection = getConnection(_positiveFailoverUrl);
         assertProducingConsuming(secondConnection);
         secondConnection.close();
     }
@@ -98,12 +88,12 @@ public class TwoNodeTest extends QpidBrokerTestCase
     public void testClusterRestartWithoutDesignatedPrimary() throws Exception
     {
         startCluster(false);
-        final Connection initialConnection = getConnection(_brokerFailoverUrl);
+        final Connection initialConnection = getConnection(_positiveFailoverUrl);
         assertProducingConsuming(initialConnection);
         initialConnection.close();
         _groupCreator.stopCluster();
         _groupCreator.startClusterParallel();
-        final Connection secondConnection = getConnection(_brokerFailoverUrl);
+        final Connection secondConnection = getConnection(_positiveFailoverUrl);
         assertProducingConsuming(secondConnection);
         secondConnection.close();
     }
@@ -112,7 +102,7 @@ public class TwoNodeTest extends QpidBrokerTestCase
     {
         startCluster(true);
         _groupCreator.stopNode(_groupCreator.getBrokerPortNumberOfSecondaryNode());
-        final Connection connection = getConnection(_brokerFailoverUrl);
+        final Connection connection = getConnection(_positiveFailoverUrl);
         assertNotNull("Expected to get a valid connection to primary", connection);
         assertProducingConsuming(connection);
     }
@@ -124,7 +114,7 @@ public class TwoNodeTest extends QpidBrokerTestCase
 
         try
         {
-            Connection connection = getConnection(_brokerFailoverUrl);
+            Connection connection = getConnection(_negativeFailoverUrl);
             assertProducingConsuming(connection);
             fail("Exception not thrown");
         }
@@ -143,7 +133,7 @@ public class TwoNodeTest extends QpidBrokerTestCase
 
         try
         {
-            getConnection(_brokerFailoverUrl);
+            getConnection(_negativeFailoverUrl);
             fail("Connection not expected");
         }
         catch (JMSException e)
@@ -155,41 +145,39 @@ public class TwoNodeTest extends QpidBrokerTestCase
     public void testInitialDesignatedPrimaryStateOfNodes() throws Exception
     {
         startCluster(true);
-        final ManagedBDBHAMessageStore primaryStoreBean = getStoreBeanForNodeAtBrokerPort(_groupCreator.getBrokerPortNumberOfPrimary());
-        assertTrue("Expected primary node to be set as designated primary", primaryStoreBean.getDesignatedPrimary());
 
-        final ManagedBDBHAMessageStore secondaryStoreBean = getStoreBeanForNodeAtBrokerPort(_groupCreator.getBrokerPortNumberOfSecondaryNode());
-        assertFalse("Expected secondary node to NOT be set as designated primary", secondaryStoreBean.getDesignatedPrimary());
+        Map<String, Object> primaryNodeAttributes = _groupCreator.getNodeAttributes(_groupCreator.getBrokerPortNumberOfPrimary());
+        assertTrue("Expected primary node to be set as designated primary",
+                   (Boolean) primaryNodeAttributes.get(BDBHAVirtualHostNode.DESIGNATED_PRIMARY));
+
+        Map<String, Object> secondaryNodeAttributes = _groupCreator.getNodeAttributes(_groupCreator.getBrokerPortNumberOfSecondaryNode());
+        assertFalse("Expected secondary node to NOT be set as designated primary",
+                    (Boolean) secondaryNodeAttributes.get(BDBHAVirtualHostNode.DESIGNATED_PRIMARY));
     }
 
     public void testSecondaryDesignatedAsPrimaryAfterOriginalPrimaryStopped() throws Exception
     {
         startCluster(true);
-        final ManagedBDBHAMessageStore storeBean = getStoreBeanForNodeAtBrokerPort(_groupCreator.getBrokerPortNumberOfSecondaryNode());
+
         _groupCreator.stopNode(_groupCreator.getBrokerPortNumberOfPrimary());
 
-        assertFalse("Expected node to NOT be set as designated primary", storeBean.getDesignatedPrimary());
-        storeBean.setDesignatedPrimary(true);
+        Map<String, Object> secondaryNodeAttributes = _groupCreator.getNodeAttributes(_groupCreator.getBrokerPortNumberOfSecondaryNode());
+        assertFalse("Expected node to NOT be set as designated primary", (Boolean) secondaryNodeAttributes.get(BDBHAVirtualHostNode.DESIGNATED_PRIMARY));
+
+        _groupCreator.setNodeAttributes(_groupCreator.getBrokerPortNumberOfSecondaryNode(), Collections.<String, Object>singletonMap(BDBHAVirtualHostNode.DESIGNATED_PRIMARY, true));
 
-        long limit = System.currentTimeMillis() + 5000;
-        while( !storeBean.getDesignatedPrimary() && System.currentTimeMillis() < limit)
+        int timeout = 5000;
+        long limit = System.currentTimeMillis() + timeout;
+        while( !((Boolean)secondaryNodeAttributes.get(BDBHAVirtualHostNode.DESIGNATED_PRIMARY)) && System.currentTimeMillis() < limit)
         {
             Thread.sleep(100);
+            secondaryNodeAttributes = _groupCreator.getNodeAttributes(_groupCreator.getBrokerPortNumberOfSecondaryNode());
         }
-        assertTrue("Expected node to now be set as designated primary", storeBean.getDesignatedPrimary());
+        assertTrue("Expected secondary to transition to primary within " + timeout, (Boolean) secondaryNodeAttributes.get(BDBHAVirtualHostNode.DESIGNATED_PRIMARY));
 
-        final Connection connection = getConnection(_brokerFailoverUrl);
+        final Connection connection = getConnection(_positiveFailoverUrl);
         assertNotNull("Expected to get a valid connection to new primary", connection);
         assertProducingConsuming(connection);
     }
 
-    private ManagedBDBHAMessageStore getStoreBeanForNodeAtBrokerPort(
-            final int activeBrokerPortNumber) throws Exception
-    {
-        _jmxUtils.open(activeBrokerPortNumber);
-
-        ManagedBDBHAMessageStore storeBean = _jmxUtils.getManagedObject(ManagedBDBHAMessageStore.class, MANAGED_OBJECT_QUERY);
-        return storeBean;
-    }
-
 }