QPID-4555: HA Test bugs causing sporadic faiulres in ha_tests.ReplicationTests.test_auto_delete_timeout

The tests were not waiting for the cluster to be ready before starting.
Updated HaCluster to wait by default before returning.

Increase timeout in calls to wait_no_queue, the default timeout of 1 sec was the same
as the auto-delete timeout.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1441157 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 19 insertions, 12 deletions

diff --git a/qpid/cpp/src/tests/ha_test.py b/qpid/cpp/src/tests/ha_test.py
index 4efbfdba3d..cdf9d3dd17 100755
--- a/qpid/cpp/src/tests/ha_test.py
+++ b/qpid/cpp/src/tests/ha_test.py
@@ -99,7 +99,8 @@ class HaBroker(Broker):
             args = args + ["--sasl-mechanism", cred.mechanism]
         self.qpid_ha_script.main_except(["", "-b", url]+args)
 
-    def promote(self): self.ready(); self.qpid_ha(["promote"])
+    def promote(self):
+        self.ready(); self.qpid_ha(["promote"])
     def set_public_url(self, url): self.qpid_ha(["set", "--public-url", url])
     def set_brokers_url(self, url): self.qpid_ha(["set", "--brokers-url", url])
     def replicate(self, from_broker, queue): self.qpid_ha(["replicate", from_broker, queue])
@@ -208,8 +209,14 @@ class HaBroker(Broker):
 class HaCluster(object):
     _cluster_count = 0
 
-    def __init__(self, test, n, promote=True, **kwargs):
-        """Start a cluster of n brokers"""
+    def __init__(self, test, n, promote=True, wait=True, **kwargs):
+        """Start a cluster of n brokers.
+
+        @test: The test being run
+        @n: start n brokers
+        @promote: promote self[0] to primary
+        @wait: wait for primary active and backups ready. Ignored if promote=False
+        """
         self.test = test
         self.kwargs = kwargs
         self._brokers = []
@@ -218,7 +225,12 @@ class HaCluster(object):
         HaCluster._cluster_count += 1
         for i in xrange(n): self.start(False)
         self.update_urls()
-        if promote: self[0].promote()
+        if promote:
+            self[0].promote()
+            if wait:
+                self[0].wait_status("active")
+                for b in self[1:]: b.wait_status("ready")
+
 
     def next_name(self):
         name="cluster%s-%s"%(self.id, self.broker_id)
diff --git a/qpid/cpp/src/tests/ha_tests.py b/qpid/cpp/src/tests/ha_tests.py
index 1725c594de..e517ce4527 100755
--- a/qpid/cpp/src/tests/ha_tests.py
+++ b/qpid/cpp/src/tests/ha_tests.py
@@ -270,7 +270,6 @@ class ReplicationTests(HaBrokerTest):
     def test_qpid_config_replication(self):
         """Set up replication via qpid-config"""
         brokers = HaCluster(self,2)
-        brokers[0].wait_status("active")
         brokers[0].config_declare("q","all")
         brokers[0].connect().session().sender("q").send("foo")
         brokers[1].assert_browse_backup("q", ["foo"])
@@ -613,8 +612,6 @@ acl deny all all
         to new members of a cluster. """
         cluster = HaCluster(self, 2)
         s = cluster[0].connect().session()
-        cluster[0].wait_status("active")
-        cluster[1].wait_status("ready")
         # altex exchange: acts as alternate exchange
         s.sender("altex;{create:always,node:{type:topic,x-declare:{type:'fanout'}}}")
         # altq queue bound to altex, collect re-routed messages.
@@ -722,7 +719,6 @@ acl deny all all
         """Regression test for QPID-4285: on deleting a queue it gets stuck in a
         partially deleted state and causes replication errors."""
         cluster = HaCluster(self,2)
-        cluster[1].wait_status("ready")
         s = cluster[0].connect().session()
         s.receiver("q;{create:always}")
         cluster[1].wait_backup("q")
@@ -804,15 +800,14 @@ acl deny all all
         cluster[1].wait_queue("q1")
         cluster[0].kill()
         cluster[1].wait_queue("q1")    # Not timed out yet
-        cluster[1].wait_no_queue("q1") # Wait for timeout
-        cluster[1].wait_no_queue("q0")
+        cluster[1].wait_no_queue("q1", timeout=5) # Wait for timeout
+        cluster[1].wait_no_queue("q0", timeout=5) # Wait for timeout
 
     def test_alt_exchange_dup(self):
         """QPID-4349: if a queue has an alterante exchange and is deleted the
         messages appear twice on the alternate, they are rerouted once by the
         primary and again by the backup."""
         cluster = HaCluster(self,2)
-        cluster[0].wait_status("active")
 
         # Set up q with alternate exchange altex bound to altq.
         s = cluster[0].connect().session()
@@ -842,7 +837,7 @@ acl deny all all
         cluster = HaCluster(self, 2)
         s = cluster[0].connect().session()
         s.sender("keep;{create:always}") # Leave this queue in place.
-        for i in xrange(100):            # FIXME aconway 2012-10-23: ??? IS this an issue?
+        for i in xrange(1000):
             s.sender("deleteme%s;{create:always,delete:always}"%(i)).close()
         # It is possible for the backup to attempt to subscribe after the queue
         # is deleted. This is not an error, but is logged as an error on the primary.