QPID-4006: BDB HA. Make passivation async to avoid holding up the BDB thread. Introduce VirtualHost ERROR state to be used when virtual host is unable to activate or passivate itself completely. Change MULTISYNC mode to use WRITE_NO_SYNC. Stop relying on Monitor nodes to perform some tests.

Work of Robbie Gemmell <robbie@apache.org> and myself.

squashme

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk/qpid@1341584 13f79535-47bb-0310-9956-ffa450edef68

3 files changed, 53 insertions, 16 deletions

diff --git a/java/broker/src/main/java/org/apache/qpid/server/logging/messages/VirtualHost_logmessages.properties b/java/broker/src/main/java/org/apache/qpid/server/logging/messages/VirtualHost_logmessages.properties
index 3e640c7929..5695026cbc 100644
--- a/java/broker/src/main/java/org/apache/qpid/server/logging/messages/VirtualHost_logmessages.properties
+++ b/java/broker/src/main/java/org/apache/qpid/server/logging/messages/VirtualHost_logmessages.properties
@@ -23,4 +23,6 @@ CREATED = VHT-1001 : Created : {0}
 CLOSED = VHT-1002 : Closed
 
 STATS_DATA = VHT-1003 : {0} : {1,choice,0#delivered|1#received} : {2,number,#.###} kB/s peak : {3,number,#} bytes total
-STATS_MSGS = VHT-1004 : {0} : {1,choice,0#delivered|1#received} : {2,number,#.###} msg/s peak : {3,number,#} msgs total`
\ No newline at end of file
+STATS_MSGS = VHT-1004 : {0} : {1,choice,0#delivered|1#received} : {2,number,#.###} msg/s peak : {3,number,#} msgs total
+
+ERRORED = VHT-1005 : Unexpected fatal error
\ No newline at end of file
diff --git a/java/broker/src/main/java/org/apache/qpid/server/virtualhost/State.java b/java/broker/src/main/java/org/apache/qpid/server/virtualhost/State.java
index fb50b3e289..55e2539dcf 100644
--- a/java/broker/src/main/java/org/apache/qpid/server/virtualhost/State.java
+++ b/java/broker/src/main/java/org/apache/qpid/server/virtualhost/State.java
@@ -25,5 +25,7 @@ public enum State
     INITIALISING,
     ACTIVE,
     PASSIVE,
-    STOPPED
+    STOPPED,
+    /** Terminal state that signifies the virtual host has experienced an unexpected condition. */
+    ERRORED
 }
diff --git a/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHostImpl.java b/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHostImpl.java
index 5a14092930..5a56fe1765 100644
--- a/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHostImpl.java
+++ b/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHostImpl.java
@@ -122,7 +122,7 @@ public class VirtualHostImpl implements VirtualHost, IConnectionRegistry.Registr
 
     private final MessageStore _messageStore;
 
-    private State _state = State.INITIALISING;
+    private volatile State _state = State.INITIALISING;
 
     private boolean _statisticsEnabled = false;
 
@@ -824,17 +824,25 @@ public class VirtualHostImpl implements VirtualHost, IConnectionRegistry.Registr
        @Override
        public void event(Event event)
        {
-           initialiseHouseKeeping(_vhostConfig.getHousekeepingCheckPeriod());
+           State finalState = State.ERRORED;
            try
            {
-               _brokerMBean.register();
+               initialiseHouseKeeping(_vhostConfig.getHousekeepingCheckPeriod());
+               try
+               {
+                   _brokerMBean.register();
+               }
+               catch (JMException e)
+               {
+                   throw new RuntimeException("Failed to register virtual host mbean for virtual host " + getName(), e);
+               }
+               finalState = State.ACTIVE;
            }
-           catch (JMException e)
+           finally
            {
-               throw new RuntimeException("Failed to register virtual host mbean for virtual host " + getName(), e);
+               _state = finalState;
+               reportIfError(_state);
            }
-
-           _state = State.ACTIVE;
        }
    }
 
@@ -842,16 +850,33 @@ public class VirtualHostImpl implements VirtualHost, IConnectionRegistry.Registr
     {
         public void event(Event event)
         {
-            _connectionRegistry.close(IConnectionRegistry.VHOST_PASSIVATE_REPLY_TEXT);
-            _brokerMBean.unregister();
-            removeHouseKeepingTasks();
+            State finalState = State.ERRORED;
+
+            try
+            {
+                /* the approach here is not ideal as there is a race condition where a
+                 * queue etc could be created while the virtual host is on the way to
+                 * the passivated state.  However the store state change from MASTER to UNKNOWN
+                 * is documented as exceptionally rare..
+                 */
+
+                _connectionRegistry.close(IConnectionRegistry.VHOST_PASSIVATE_REPLY_TEXT);
+                _brokerMBean.unregister();
+                removeHouseKeepingTasks();
 
-            _queueRegistry.stopAllAndUnregisterMBeans();
-            _exchangeRegistry.clearAndUnregisterMbeans();
-            _dtxRegistry.close();
+                _queueRegistry.stopAllAndUnregisterMBeans();
+                _exchangeRegistry.clearAndUnregisterMbeans();
+                _dtxRegistry.close();
 
-            _state = State.PASSIVE;
+                finalState = State.PASSIVE;
+            }
+            finally
+            {
+                _state = finalState;
+                reportIfError(_state);
+            }
         }
+
     }
 
     private final class BeforeCloseListener implements EventListener
@@ -864,6 +889,14 @@ public class VirtualHostImpl implements VirtualHost, IConnectionRegistry.Registr
         }
     }
 
+    private void reportIfError(State state)
+    {
+        if (state == State.ERRORED)
+        {
+            CurrentActor.get().message(VirtualHostMessages.ERRORED());
+        }
+    }
+
     private class VirtualHostHouseKeepingTask extends HouseKeepingTask
     {
         public VirtualHostHouseKeepingTask()