QPID-5998: Fix HA operational logging

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1618086 13f79535-47bb-0310-9956-ffa450edef68

6 files changed, 218 insertions, 193 deletions

diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/logging/subjects/BDBHAVirtualHostNodeLogSubject.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/logging/subjects/BDBHAVirtualHostNodeLogSubject.java
new file mode 100644
index 0000000000..a209062993
--- /dev/null
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/logging/subjects/BDBHAVirtualHostNodeLogSubject.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.qpid.server.logging.subjects;
+
+
+public class BDBHAVirtualHostNodeLogSubject extends AbstractLogSubject
+{
+    public static final String VIRTUAL_HOST_NODE_FORMAT = "grp(/{0})/vhn(/{1})";
+
+    public BDBHAVirtualHostNodeLogSubject(String groupName, String nodeName)
+    {
+        setLogStringWithFormat(VIRTUAL_HOST_NODE_FORMAT, groupName, nodeName);
+    }
+}
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/logging/subjects/GroupLogSubject.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/logging/subjects/GroupLogSubject.java
new file mode 100644
index 0000000000..51fd1fc2dc
--- /dev/null
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/logging/subjects/GroupLogSubject.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.qpid.server.logging.subjects;
+
+
+public class GroupLogSubject extends AbstractLogSubject
+{
+    public static final String GROUP_FORMAT = "grp(/{0})";
+
+    public GroupLogSubject(String groupName)
+    {
+        setLogStringWithFormat(GROUP_FORMAT, groupName);
+    }
+}
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
index 5263f5942f..06671998ec 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
@@ -30,9 +30,14 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 import com.sleepycat.je.rep.MasterStateException;
+import com.sleepycat.je.rep.ReplicatedEnvironment;
 
 import org.apache.log4j.Logger;
 import org.apache.qpid.server.configuration.IllegalConfigurationException;
+import org.apache.qpid.server.logging.EventLogger;
+import org.apache.qpid.server.logging.messages.HighAvailabilityMessages;
+import org.apache.qpid.server.logging.subjects.BDBHAVirtualHostNodeLogSubject;
+import org.apache.qpid.server.logging.subjects.GroupLogSubject;
 import org.apache.qpid.server.model.AbstractConfiguredObject;
 import org.apache.qpid.server.model.Broker;
 import org.apache.qpid.server.model.ConfiguredObject;
@@ -40,6 +45,8 @@ import org.apache.qpid.server.model.IllegalStateTransitionException;
 import org.apache.qpid.server.model.ManagedAttributeField;
 import org.apache.qpid.server.model.State;
 import org.apache.qpid.server.model.StateTransition;
+import org.apache.qpid.server.model.SystemConfig;
+import org.apache.qpid.server.model.VirtualHostNode;
 import org.apache.qpid.server.security.access.Operation;
 import org.apache.qpid.server.store.berkeleydb.replication.ReplicatedEnvironmentFacade;
 
@@ -53,13 +60,16 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
 
     private volatile long _joinTime;
     private volatile long _lastTransactionId;
+    private volatile String _lastReplicatedEnvironmentState = ReplicatedEnvironment.State.UNKNOWN.name();
 
     @ManagedAttributeField(afterSet="afterSetRole")
-    private volatile String _role;
+    private volatile String _role = ReplicatedEnvironment.State.UNKNOWN.name();
 
     private final AtomicReference<State> _state;
     private final boolean _isMonitor;
     private boolean _detached;
+    private BDBHAVirtualHostNodeLogSubject _virtualHostNodeLogSubject;
+    private GroupLogSubject _groupLogSubject;
 
     public BDBHARemoteReplicationNodeImpl(BDBHAVirtualHostNode<?> virtualHostNode, Map<String, Object> attributes, ReplicatedEnvironmentFacade replicatedEnvironmentFacade)
     {
@@ -92,7 +102,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
     @Override
     public String getRole()
     {
-        return _role;
+        return _lastReplicatedEnvironmentState;
     }
 
     @Override
@@ -152,10 +162,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
     {
         String nodeName = getName();
 
-        if (LOGGER.isDebugEnabled())
-        {
-            LOGGER.debug("Deleting node '"  + nodeName + "' from group '" + getGroupName() + "'");
-        }
+        getEventLogger().message(_virtualHostNodeLogSubject, HighAvailabilityMessages.DELETED());
 
         try
         {
@@ -178,19 +185,11 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         try
         {
             String nodeName = getName();
-            if (LOGGER.isDebugEnabled())
-            {
-                LOGGER.debug("Trying to transfer master to '" + nodeName + "'");
-            }
+            getEventLogger().message(_groupLogSubject, HighAvailabilityMessages.TRANSFER_MASTER(getName(), getAddress()));
 
             _replicatedEnvironmentFacade.transferMasterAsynchronously(nodeName);
-
-            if (LOGGER.isDebugEnabled())
-            {
-                LOGGER.debug("The transfer of mastership to node '" + nodeName + "' has been initiated.");
-            }
         }
-        catch(Exception e)
+        catch (Exception e)
         {
             throw new IllegalConfigurationException("Cannot transfer mastership to '" + getName() + "'", e);
         }
@@ -227,10 +226,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
 
     void setRole(String role)
     {
-        if (LOGGER.isDebugEnabled())
-        {
-            LOGGER.debug(this + " updating role to : " + role);
-        }
+        _lastReplicatedEnvironmentState = role;
         _role = role;
         updateModelStateFromRole(role);
     }
@@ -266,4 +262,17 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
     {
         this._detached = detached;
     }
+
+    @Override
+    public void onValidate()
+    {
+        super.onValidate();
+        _virtualHostNodeLogSubject =  new BDBHAVirtualHostNodeLogSubject(getGroupName(), getName());
+        _groupLogSubject = new GroupLogSubject(getGroupName());
+    }
+
+    private EventLogger getEventLogger()
+    {
+        return ((SystemConfig)getParent(VirtualHostNode.class).getParent(Broker.class).getParent(SystemConfig.class)).getEventLogger();
+    }
 }
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
index 5489493f74..fd098a8ef6 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
@@ -22,6 +22,7 @@ package org.apache.qpid.server.virtualhostnode.berkeleydb;
 
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
+import java.text.MessageFormat;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -49,6 +50,8 @@ import org.apache.qpid.server.configuration.IllegalConfigurationException;
 import org.apache.qpid.server.configuration.updater.Task;
 import org.apache.qpid.server.logging.messages.ConfigStoreMessages;
 import org.apache.qpid.server.logging.messages.HighAvailabilityMessages;
+import org.apache.qpid.server.logging.subjects.BDBHAVirtualHostNodeLogSubject;
+import org.apache.qpid.server.logging.subjects.GroupLogSubject;
 import org.apache.qpid.server.model.Broker;
 import org.apache.qpid.server.model.BrokerModel;
 import org.apache.qpid.server.model.ConfiguredObject;
@@ -75,6 +78,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         BDBHAVirtualHostNode<BDBHAVirtualHostNodeImpl>
 {
     public static final String VIRTUAL_HOST_NODE_TYPE = "BDB_HA";
+    public static final String VIRTUAL_HOST_PRINCIPAL_NAME_FORMAT = "grp(/{0})/vhn(/{1})";
 
     /**
      * Length of time we synchronously await the a JE mutation to complete.  It is not considered an error if we exceed this timeout, although a
@@ -87,6 +91,9 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
     private final AtomicReference<ReplicatedEnvironmentFacade> _environmentFacade = new AtomicReference<>();
 
     private final AtomicReference<ReplicatedEnvironment.State> _lastReplicatedEnvironmentState = new AtomicReference<>(ReplicatedEnvironment.State.UNKNOWN);
+    private BDBHAVirtualHostNodeLogSubject _virtualHostNodeLogSubject;
+    private GroupLogSubject _groupLogSubject;
+    private String _virtualHostNodePrincipalName;
 
     @ManagedAttributeField
     private String _storePath;
@@ -267,7 +274,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             }
         }
 
-        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ADDED(getName(), getGroupName()));
+        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.CREATED());
     }
 
     protected ReplicatedEnvironmentFacade getReplicatedEnvironmentFacade()
@@ -291,8 +298,6 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
 
         getConfigurationStore().openConfigurationStore(this, false);
 
-        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ATTACHED(getName(), getGroupName(), getRole()));
-
         getEventLogger().message(getConfigurationStoreLogSubject(), ConfigStoreMessages.CREATED());
         getEventLogger().message(getConfigurationStoreLogSubject(), ConfigStoreMessages.STORE_LOCATION(getStorePath()));
 
@@ -325,23 +330,6 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             ReplicatedEnvironment.State detached = ReplicatedEnvironment.State.DETACHED;
             _lastReplicatedEnvironmentState.set(detached);
             attributeSet(ROLE, _role, detached);
-
-            //Perhaps, having STOPPED operational logging could be sufficient. However, on START we still will be seeing 2 logs: ATTACHED and STARTED
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DETACHED(getName(), getGroupName()));
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.STOPPED(getName(), getGroupName()));
-        }
-    }
-
-    @StateTransition( currentState = { State.UNINITIALIZED, State.STOPPED, State.ERRORED }, desiredState = State.ACTIVE )
-    protected void doActivate()
-    {
-        try
-        {
-            super.doActivate();
-        }
-        finally
-        {
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.STARTED(getName(), getGroupName()));
         }
     }
 
@@ -359,7 +347,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
     {
         Set<InetSocketAddress> helpers = getRemoteNodeAddresses();
         super.doDelete();
-        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DELETED(getName(), getGroupName()));
+        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DELETED());
         if (getState() == State.DELETED && !helpers.isEmpty())
         {
             try
@@ -413,10 +401,18 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         finally
         {
             closeEnvironment();
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DETACHED(getName(), getGroupName()));
         }
     }
 
+    @Override
+    public void onValidate()
+    {
+        super.onValidate();
+        _virtualHostNodeLogSubject =  new BDBHAVirtualHostNodeLogSubject(getGroupName(), getName());
+        _groupLogSubject = new GroupLogSubject(getGroupName());
+        _virtualHostNodePrincipalName = MessageFormat.format(VIRTUAL_HOST_PRINCIPAL_NAME_FORMAT, getGroupName(), getName());
+    }
+
     private void onMaster()
     {
         try
@@ -550,7 +546,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             {
                 LOGGER.info("Received BDB event indicating transition to state " + state);
             }
-
+            String previousRole = getRole();
             try
             {
                 switch (state)
@@ -573,10 +569,9 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             finally
             {
                 _lastReplicatedEnvironmentState.set(state);
-                String previousRole = _role;
                 attributeSet(ROLE, _role, state.name());
-                getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ROLE_CHANGED(getName(),
-                        getGroupName(), previousRole, state.name()));
+                getEventLogger().message(getGroupLogSubject(),
+                        HighAvailabilityMessages.ROLE_CHANGED(getName(), getAddress(), previousRole, state.name()));
             }
         }
     }
@@ -591,8 +586,8 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             try
             {
                 environmentFacade.setPriority(_priority).get(MUTATE_JE_TIMEOUT_MS, TimeUnit.MILLISECONDS);
-                getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.PRIORITY_CHANGED(getName(),
-                        getGroupName(), String.valueOf(_priority)));
+                getEventLogger().message(getVirtualHostNodeLogSubject(),
+                        HighAvailabilityMessages.PRIORITY_CHANGED(String.valueOf(_priority)));
             }
             catch (TimeoutException e)
             {
@@ -619,8 +614,8 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             try
             {
                 environmentFacade.setDesignatedPrimary(_designatedPrimary).get(MUTATE_JE_TIMEOUT_MS, TimeUnit.MILLISECONDS);
-                getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DESIGNATED_PRIMARY_CHANGED(getName(),
-                        getGroupName(), String.valueOf(_designatedPrimary)));
+                getEventLogger().message(getVirtualHostNodeLogSubject(),
+                        HighAvailabilityMessages.DESIGNATED_PRIMARY_CHANGED(String.valueOf(_designatedPrimary)));
             }
             catch (TimeoutException e)
             {
@@ -647,8 +642,8 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             try
             {
                 environmentFacade.setElectableGroupSizeOverride(_quorumOverride).get(MUTATE_JE_TIMEOUT_MS, TimeUnit.MILLISECONDS);
-                getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.QUORUM_OVERRIDE_CHANGED(getName(),
-                        getGroupName(), String.valueOf(_quorumOverride)));
+                getEventLogger().message(getVirtualHostNodeLogSubject(),
+                        HighAvailabilityMessages.QUORUM_OVERRIDE_CHANGED(String.valueOf(_quorumOverride)));
             }
             catch (TimeoutException e)
             {
@@ -674,8 +669,8 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         {
             try
             {
+                getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.TRANSFER_MASTER(getName(), getAddress()));
                 environmentFacade.transferMasterToSelfAsynchronously().get(MUTATE_JE_TIMEOUT_MS, TimeUnit.MILLISECONDS);
-                getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.TRANSFER_MASTER(getName(), getName(), getGroupName()));
             }
             catch (TimeoutException e)
             {
@@ -701,18 +696,27 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         return getAddress().equals(getHelperAddress());
     }
 
+    BDBHAVirtualHostNodeLogSubject getVirtualHostNodeLogSubject()
+    {
+        return _virtualHostNodeLogSubject;
+    }
+
+    GroupLogSubject getGroupLogSubject()
+    {
+        return _groupLogSubject;
+    }
+
     private class RemoteNodesDiscoverer implements ReplicationGroupListener
     {
         @Override
         public void onReplicationNodeAddedToGroup(final ReplicationNode node)
         {
-            getTaskExecutor().submit(new Task<Void>()
+            getTaskExecutor().submit(new VirtualHostNodeGroupTask()
             {
                 @Override
-                public Void execute()
+                public void perform()
                 {
                     addRemoteReplicationNode(node);
-                    return null;
                 }
             });
         }
@@ -722,19 +726,18 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             BDBHARemoteReplicationNodeImpl remoteNode = new BDBHARemoteReplicationNodeImpl(BDBHAVirtualHostNodeImpl.this, nodeToAttributes(node), getReplicatedEnvironmentFacade());
             remoteNode.create();
             childAdded(remoteNode);
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ADDED(remoteNode.getName(), getGroupName()));
+            getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.ADDED(remoteNode.getName(), remoteNode.getAddress()));
         }
 
         @Override
         public void onReplicationNodeRecovered(final ReplicationNode node)
         {
-            getTaskExecutor().submit(new Task<Void>()
+            getTaskExecutor().submit(new VirtualHostNodeGroupTask()
             {
                 @Override
-                public Void execute()
+                public void perform()
                 {
                     recoverRemoteReplicationNode(node);
-                    return null;
                 }
             });
         }
@@ -745,19 +748,18 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             remoteNode.registerWithParents();
             remoteNode.open();
 
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ATTACHED(remoteNode.getName(), getGroupName(), String.valueOf(remoteNode.getState())));
+            getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.JOINED(remoteNode.getName(), remoteNode.getAddress()));
         }
 
         @Override
         public void onReplicationNodeRemovedFromGroup(final ReplicationNode node)
         {
-            getTaskExecutor().submit(new Task<Void>()
+            getTaskExecutor().submit(new VirtualHostNodeGroupTask()
             {
                 @Override
-                public Void execute()
+                public void perform()
                 {
                     removeRemoteReplicationNode(node);
-                    return null;
                 }
             });
         }
@@ -768,12 +770,25 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             if (remoteNode != null)
             {
                 remoteNode.deleted();
-                getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DELETED(remoteNode.getName(), getGroupName()));
+                getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.REMOVED(remoteNode.getName(), remoteNode.getAddress()));
             }
         }
 
         @Override
-        public void onNodeState(ReplicationNode node, NodeState nodeState)
+        public void onNodeState(final ReplicationNode node, final NodeState nodeState)
+        {
+            Subject.doAs(SecurityManager.getSystemTaskSubject(_virtualHostNodePrincipalName), new PrivilegedAction<Void>()
+            {
+                @Override
+                public Void run()
+                {
+                    processNodeState(node, nodeState);
+                    return null;
+                }
+            });
+        }
+
+        private void processNodeState(ReplicationNode node, NodeState nodeState)
         {
             BDBHARemoteReplicationNodeImpl remoteNode = getChildByName(BDBHARemoteReplicationNodeImpl.class, node.getName());
             if (remoteNode != null)
@@ -785,7 +800,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
                     remoteNode.setLastTransactionId(-1);
                     if (!remoteNode.isDetached())
                     {
-                        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DETACHED(remoteNode.getName(), getGroupName()));
+                        getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.LEFT(remoteNode.getName(), remoteNode.getAddress()));
                         remoteNode.setDetached(true);
                     }
                 }
@@ -796,7 +811,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
                     remoteNode.setRole(nodeState.getNodeState().name());
                     if (remoteNode.isDetached())
                     {
-                        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ATTACHED(remoteNode.getName(), getGroupName(), remoteNode.getRole() ));
+                        getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.JOINED(remoteNode.getName(), remoteNode.getAddress()));
                         remoteNode.setDetached(false);
                     }
                 }
@@ -804,17 +819,29 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
                 String newRole = remoteNode.getRole();
                 if (!newRole.equals(currentRole))
                 {
-                    getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ROLE_CHANGED(remoteNode.getName(),
-                            getGroupName(), currentRole, newRole));
+                    getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.ROLE_CHANGED(remoteNode.getName(), remoteNode.getAddress(), currentRole, newRole));
                 }
             }
         }
 
         @Override
-        public void onIntruderNode(ReplicationNode node)
+        public void onIntruderNode(final ReplicationNode node)
+        {
+            Subject.doAs(SecurityManager.getSystemTaskSubject(_virtualHostNodePrincipalName), new PrivilegedAction<Void>()
+            {
+                @Override
+                public Void run()
+                {
+                    processIntruderNode(node);
+                    return null;
+                }
+            });
+        }
+
+        private void processIntruderNode(ReplicationNode node)
         {
             String hostAndPort = node.getHostName() + ":" + node.getPort();
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.INTRUDER_DETECTED(node.getName(), hostAndPort, getGroupName()));
+            getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.INTRUDER_DETECTED(node.getName(), hostAndPort));
 
             boolean inManagementMode = getParent(Broker.class).isManagementMode();
             if (inManagementMode)
@@ -858,7 +885,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         @Override
         public void onNoMajority()
         {
-            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.MAJORITY_LOST(getName(), getGroupName()));
+            getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.QUORUM_LOST());
         }
 
         private Map<String, Object> nodeToAttributes(ReplicationNode replicationNode)
@@ -872,4 +899,23 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         }
     }
 
+    private abstract class VirtualHostNodeGroupTask implements Task<Void>
+    {
+        @Override
+        public Void execute()
+        {
+            return Subject.doAs(SecurityManager.getSystemTaskSubject(_virtualHostNodePrincipalName), new PrivilegedAction<Void>()
+            {
+                @Override
+                public Void run()
+                {
+                    perform();
+                    return null;
+                }
+            });
+        }
+
+        abstract void perform();
+    }
+
 }
diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeTest.java
index 6259b49d61..0d64d87aef 100644
--- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeTest.java
+++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeTest.java
@@ -19,6 +19,7 @@
 
 package org.apache.qpid.server.virtualhostnode.berkeleydb;
 
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -33,6 +34,7 @@ import org.apache.qpid.server.configuration.updater.TaskExecutor;
 import org.apache.qpid.server.model.Broker;
 import org.apache.qpid.server.model.ConfiguredObjectFactory;
 import org.apache.qpid.server.model.VirtualHost;
+import org.apache.qpid.server.model.VirtualHostNode;
 import org.apache.qpid.server.security.SecurityManager;
 import org.apache.qpid.server.security.access.Operation;
 import org.apache.qpid.server.store.DurableConfigurationStore;
@@ -69,7 +71,7 @@ public class BDBHARemoteReplicationNodeTest extends QpidTestCase
 
         // Virtualhost needs the EventLogger from the SystemContext.
         when(_virtualHostNode.getParent(Broker.class)).thenReturn(_broker);
-
+        doReturn(VirtualHostNode.class).when(_virtualHostNode).getCategoryClass();
         ConfiguredObjectFactory objectFactory = _broker.getObjectFactory();
         when(_virtualHostNode.getModel()).thenReturn(objectFactory.getModel());
         when(_virtualHostNode.getTaskExecutor()).thenReturn(_taskExecutor);
@@ -80,7 +82,7 @@ public class BDBHARemoteReplicationNodeTest extends QpidTestCase
         String remoteReplicationName = getName();
         BDBHARemoteReplicationNode remoteReplicationNode = createRemoteReplicationNode(remoteReplicationName);
 
-        remoteReplicationNode.setAttribute(BDBHARemoteReplicationNode.ROLE, null, "MASTER");
+        remoteReplicationNode.setAttribute(BDBHARemoteReplicationNode.ROLE, "UNKNOWN", "MASTER");
 
         verify(_facade).transferMasterAsynchronously(remoteReplicationName);
     }
diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java
index ef1021160c..ea7d74090d 100644
--- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java
+++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java
@@ -78,67 +78,18 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
 
         _helper.assertNodeRole(node1, "MASTER");
 
-        String expectedMessage = HighAvailabilityMessages.ADDED(node1.getName(), node1.getGroupName()).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.ADDED_LOG_HIERARCHY)));
-
-        expectedMessage = HighAvailabilityMessages.ATTACHED(node1.getName(), node1.getGroupName(), "UNKNOWN").toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.ATTACHED_LOG_HIERARCHY)));
+        assertEquals("Unexpected VHN log subject", "[grp(/group)/vhn(/node1)] ", node1.getVirtualHostNodeLogSubject().getLogString());
+        assertEquals("Unexpected group log subject", "[grp(/group)] ", node1.getGroupLogSubject().getLogString());
 
-
-        expectedMessage = HighAvailabilityMessages.STARTED(node1.getName(), node1.getGroupName()).toString();
+        String expectedMessage = HighAvailabilityMessages.CREATED().toString();
         verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.STARTED_LOG_HIERARCHY)));
+                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.CREATED_LOG_HIERARCHY)));
 
-        expectedMessage = HighAvailabilityMessages.ROLE_CHANGED(node1.getName(), node1.getGroupName(), "UNKNOWN", "MASTER").toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
+        expectedMessage = HighAvailabilityMessages.ROLE_CHANGED(node1.getName(), node1.getAddress(), "UNKNOWN", "MASTER").toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getGroupLogSubject())),
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.ROLE_CHANGED_LOG_HIERARCHY)));
     }
 
-    public void testStop() throws Exception
-    {
-        int node1PortNumber = findFreePort();
-        String helperAddress = "localhost:" + node1PortNumber;
-        String groupName = "group";
-        String nodeName = "node1";
-
-        Map<String, Object> node1Attributes = _helper.createNodeAttributes(nodeName, groupName, helperAddress, helperAddress, nodeName, node1PortNumber);
-        BDBHAVirtualHostNodeImpl node1 = (BDBHAVirtualHostNodeImpl)_helper.createHaVHN(node1Attributes);
-        _helper.assertNodeRole(node1, "MASTER");
-        reset(_eventLogger);
-
-        node1.stop();
-
-        String expectedMessage = HighAvailabilityMessages.DETACHED(node1.getName(), node1.getGroupName()).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DETACHED_LOG_HIERARCHY)));
-
-        expectedMessage = HighAvailabilityMessages.STOPPED(node1.getName(), node1.getGroupName()).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.STOPPED_LOG_HIERARCHY)));
-    }
-
-    public void testClose() throws Exception
-    {
-        int node1PortNumber = findFreePort();
-        String helperAddress = "localhost:" + node1PortNumber;
-        String groupName = "group";
-        String nodeName = "node1";
-
-        Map<String, Object> node1Attributes = _helper.createNodeAttributes(nodeName, groupName, helperAddress, helperAddress, nodeName, node1PortNumber);
-        BDBHAVirtualHostNodeImpl node1 = (BDBHAVirtualHostNodeImpl)_helper.createHaVHN(node1Attributes);
-        _helper.assertNodeRole(node1, "MASTER");
-
-        reset(_eventLogger);
-
-        node1.close();
-
-        String expectedMessage = HighAvailabilityMessages.DETACHED(node1.getName(), node1.getGroupName()).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DETACHED_LOG_HIERARCHY)));
-    }
-
     public void testDelete() throws Exception
     {
         int node1PortNumber = findFreePort();
@@ -154,13 +105,10 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
 
         node1.delete();
 
-        String expectedMessage = HighAvailabilityMessages.DETACHED(node1.getName(), node1.getGroupName()).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DETACHED_LOG_HIERARCHY)));
-
-        expectedMessage = HighAvailabilityMessages.DELETED(node1.getName(), node1.getGroupName()).toString();
+        String expectedMessage = HighAvailabilityMessages.DELETED().toString();
         verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DELETED_LOG_HIERARCHY)));
+
     }
 
     public void testSetPriority() throws Exception
@@ -181,7 +129,7 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         // make sure that task executor thread finishes all scheduled tasks
         node1.stop();
 
-        String expectedMessage = HighAvailabilityMessages.PRIORITY_CHANGED(node1.getName(), node1.getGroupName(), "10").toString();
+        String expectedMessage = HighAvailabilityMessages.PRIORITY_CHANGED("10").toString();
         verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.PRIORITY_CHANGED_LOG_HIERARCHY)));
     }
@@ -204,7 +152,7 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         // make sure that task executor thread finishes all scheduled tasks
         node1.stop();
 
-        String expectedMessage = HighAvailabilityMessages.QUORUM_OVERRIDE_CHANGED(node1.getName(), node1.getGroupName(), "1").toString();
+        String expectedMessage = HighAvailabilityMessages.QUORUM_OVERRIDE_CHANGED("1").toString();
         verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.QUORUM_OVERRIDE_CHANGED_LOG_HIERARCHY)));
     }
@@ -227,7 +175,7 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         // make sure that task executor thread finishes all scheduled tasks
         node1.stop();
 
-        String expectedMessage = HighAvailabilityMessages.DESIGNATED_PRIMARY_CHANGED(node1.getName(), node1.getGroupName(), "true").toString();
+        String expectedMessage = HighAvailabilityMessages.DESIGNATED_PRIMARY_CHANGED("true").toString();
         verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DESIGNATED_PRIMARY_CHANGED_LOG_HIERARCHY)));
     }
@@ -254,14 +202,9 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         // make sure that task executor thread finishes all scheduled tasks
         node2.stop();
 
-        // Verify ADDED message from node2 when its created
-        String expectedMessage = HighAvailabilityMessages.ADDED(node2.getName(), groupName).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node2.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.ADDED_LOG_HIERARCHY)));
-
         // Verify ADDED message from node1 when it discovers node2 has been added
-        expectedMessage = HighAvailabilityMessages.ADDED(node2.getName(), groupName).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
+        String expectedMessage = HighAvailabilityMessages.ADDED(node2.getName(), node2.getAddress()).toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getGroupLogSubject())),
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.ADDED_LOG_HIERARCHY)));
     }
 
@@ -292,9 +235,9 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         // make sure that task executor thread finishes all scheduled tasks
         node1.stop();
 
-        String expectedMessage = HighAvailabilityMessages.DELETED(node2.getName(), groupName).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DELETED_LOG_HIERARCHY)));
+        String expectedMessage = HighAvailabilityMessages.REMOVED(node2.getName(), node2.getAddress()).toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getGroupLogSubject())),
+                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.REMOVED_LOG_HIERARCHY)));
     }
 
     public void testRemoteNodeDetached() throws Exception
@@ -324,9 +267,9 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         waitForNodeDetachedField(remoteNode, true);
 
         // verify that remaining node issues the DETACHED operational logging for remote node
-        String expectedMessage = HighAvailabilityMessages.DETACHED(node2.getName(), groupName).toString();
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DETACHED_LOG_HIERARCHY)));
+        String expectedMessage = HighAvailabilityMessages.LEFT(node2.getName(), node2.getAddress()).toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getGroupLogSubject())),
+                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.LEFT_LOG_HIERARCHY)));
     }
 
 
@@ -361,48 +304,9 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
         _helper.assertNodeRole(node2, "REPLICA", "MASTER");
         waitForNodeDetachedField(remoteNode, false);
 
-        final String expectedMessage = HighAvailabilityMessages.ATTACHED(node2.getName(), groupName, "REPLICA").toString();
-        final String expectedMessage2 = HighAvailabilityMessages.ATTACHED(node2.getName(), groupName, "UNKNOWN").toString();
-        final String expectedMessage3 = HighAvailabilityMessages.ATTACHED(node2.getName(), groupName, "MASTER").toString();
-        ArgumentMatcher<LogMessage> matcher = new ArgumentMatcher<LogMessage>()
-        {
-            private String _messageErrorDescription = null;
-            private String _hierarchyErrorDescription = null;
-
-            @Override
-            public boolean matches(Object argument)
-            {
-                LogMessage logMessage = (LogMessage)argument;
-                String actualMessage = logMessage.toString();
-                boolean expectedMessageMatches = expectedMessage.equals(actualMessage)
-                        || expectedMessage2.equals(actualMessage) || expectedMessage3.equals(actualMessage);
-                if (!expectedMessageMatches)
-                {
-                    _messageErrorDescription = "Actual message does not match any expected: " + actualMessage;
-                }
-                boolean expectedHierarchyMatches = HighAvailabilityMessages.ATTACHED_LOG_HIERARCHY.equals(logMessage.getLogHierarchy());
-                if (!expectedHierarchyMatches)
-                {
-                    _hierarchyErrorDescription = "Actual hierarchy does not match expected: " + logMessage.getLogHierarchy();
-                }
-                return expectedMessageMatches && expectedHierarchyMatches;
-            }
-
-            @Override
-            public void describeTo(Description description)
-            {
-                if (_messageErrorDescription != null)
-                {
-                    description.appendText(_messageErrorDescription);
-                }
-                if (_hierarchyErrorDescription != null)
-                {
-                    description.appendText(_hierarchyErrorDescription);
-                }
-            }
-        };
-        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
-                argThat(matcher));
+        final String expectedMessage = HighAvailabilityMessages.JOINED(node2.getName(), node2.getAddress()).toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getGroupLogSubject())),
+                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.JOINED_LOG_HIERARCHY)));
     }
 
     private void waitForNodeDetachedField(BDBHARemoteReplicationNodeImpl remoteNode, boolean expectedDetached) throws InterruptedException {