QPID-5891: Add operational logging for the cases when remote node becomes detached or attached

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1612321 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 118 insertions, 0 deletions

diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
index 64b29b8daf..5263f5942f 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
@@ -59,6 +59,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
 
     private final AtomicReference<State> _state;
     private final boolean _isMonitor;
+    private boolean _detached;
 
     public BDBHARemoteReplicationNodeImpl(BDBHAVirtualHostNode<?> virtualHostNode, Map<String, Object> attributes, ReplicatedEnvironmentFacade replicatedEnvironmentFacade)
     {
@@ -255,4 +256,14 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         boolean isActive = MASTER.name().equals(role) || REPLICA.name().equals(role);
         _state.compareAndSet(currentState, isActive ? State.ACTIVE : State.UNAVAILABLE);
     }
+
+    public boolean isDetached()
+    {
+        return _detached;
+    }
+
+    public void setDetached(boolean detached)
+    {
+        this._detached = detached;
+    }
 }
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
index e629da3d26..8b3b45c63f 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
@@ -757,12 +757,22 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
                 if (nodeState == null)
                 {
                     remoteNode.setRole(ReplicatedEnvironment.State.UNKNOWN.name());
+                    if (!remoteNode.isDetached())
+                    {
+                        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.DETACHED(remoteNode.getName(), getGroupName()));
+                        remoteNode.setDetached(true);
+                    }
                 }
                 else
                 {
                     remoteNode.setJoinTime(nodeState.getJoinTime());
                     remoteNode.setLastTransactionId(nodeState.getCurrentTxnEndVLSN());
                     remoteNode.setRole(nodeState.getNodeState().name());
+                    if (remoteNode.isDetached())
+                    {
+                        getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.ATTACHED(remoteNode.getName(), getGroupName(), remoteNode.getRole() ));
+                        remoteNode.setDetached(false);
+                    }
                 }
 
                 String newRole = remoteNode.getRole();
diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java
index e39ca0d740..e73d4a27fd 100644
--- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java
+++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeOperationalLoggingTest.java
@@ -284,6 +284,86 @@ public class BDBHAVirtualHostNodeOperationalLoggingTest extends QpidTestCase
                 argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DELETED_LOG_HIERARCHY)));
     }
 
+    public void testRemoteNodeDetached() throws Exception
+    {
+        int node1PortNumber = findFreePort();
+        String helperAddress = "localhost:" + node1PortNumber;
+        String groupName = "group";
+        String nodeName = "node1";
+
+        Map<String, Object> node1Attributes = _helper.createNodeAttributes(nodeName, groupName, helperAddress, helperAddress);
+        node1Attributes.put(BDBHAVirtualHostNode.DESIGNATED_PRIMARY, true);
+        BDBHAVirtualHostNodeImpl node1 = (BDBHAVirtualHostNodeImpl)_helper.createHaVHN(node1Attributes);
+        _helper.assertNodeRole(node1, "MASTER");
+
+        int node2PortNumber = getNextAvailable(node1PortNumber + 1);
+        Map<String, Object> node2Attributes = _helper.createNodeAttributes("node2", groupName, "localhost:" + node2PortNumber, helperAddress);
+        BDBHAVirtualHostNodeImpl node2 = (BDBHAVirtualHostNodeImpl)_helper.createHaVHN(node2Attributes);
+        _helper.awaitRemoteNodes(node1, 1);
+
+        reset(_eventLogger);
+
+        BDBHARemoteReplicationNodeImpl remoteNode = (BDBHARemoteReplicationNodeImpl)node1.getRemoteReplicationNodes().iterator().next();
+
+        // close remote node
+        node2.close();
+
+        waitForNodeDetachedField(remoteNode, true);
+
+        // verify that remaining node issues the DETACHED operational logging for remote node
+        String expectedMessage = HighAvailabilityMessages.DETACHED(node2.getName(), groupName).toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
+                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.DETACHED_LOG_HIERARCHY)));
+    }
+
+
+    public void testRemoteNodeReAttached() throws Exception
+    {
+        int node1PortNumber = findFreePort();
+        String helperAddress = "localhost:" + node1PortNumber;
+        String groupName = "group";
+        String nodeName = "node1";
+
+        Map<String, Object> node1Attributes = _helper.createNodeAttributes(nodeName, groupName, helperAddress, helperAddress);
+        node1Attributes.put(BDBHAVirtualHostNode.DESIGNATED_PRIMARY, true);
+        BDBHAVirtualHostNodeImpl node1 = (BDBHAVirtualHostNodeImpl)_helper.createHaVHN(node1Attributes);
+        _helper.assertNodeRole(node1, "MASTER");
+
+        int node2PortNumber = getNextAvailable(node1PortNumber + 1);
+        Map<String, Object> node2Attributes = _helper.createNodeAttributes("node2", groupName, "localhost:" + node2PortNumber, helperAddress);
+        BDBHAVirtualHostNodeImpl node2 = (BDBHAVirtualHostNodeImpl)_helper.createHaVHN(node2Attributes);
+        _helper.awaitRemoteNodes(node1, 1);
+
+        BDBHARemoteReplicationNodeImpl remoteNode = (BDBHARemoteReplicationNodeImpl)node1.getRemoteReplicationNodes().iterator().next();
+
+        // stop remote node
+        node2.stop();
+
+        waitForNodeDetachedField(remoteNode, true);
+
+        reset(_eventLogger);
+        resetEventLogger();
+
+        node2 = (BDBHAVirtualHostNodeImpl)_helper.recoverHaVHN(node2.getId(), node2Attributes);
+        _helper.assertNodeRole(node2, "REPLICA");
+
+        waitForNodeDetachedField(remoteNode, false);
+
+        // verify that remaining node issues the ATTACHED operational logging for remote node
+        String expectedMessage = HighAvailabilityMessages.ATTACHED(node2.getName(), groupName, "REPLICA").toString();
+        verify(_eventLogger).message(argThat(new LogSubjectMatcher(node1.getVirtualHostNodeLogSubject())),
+                argThat(new LogMessageMatcher(expectedMessage, HighAvailabilityMessages.ATTACHED_LOG_HIERARCHY)));
+    }
+
+    private void waitForNodeDetachedField(BDBHARemoteReplicationNodeImpl remoteNode, boolean expectedDetached) throws InterruptedException {
+        int counter = 0;
+        while (expectedDetached != remoteNode.isDetached() && counter<50)
+        {
+            Thread.sleep(100);
+            counter++;
+        }
+    }
+
     private EventLogger resetEventLogger()
     {
         EventLogger eventLogger = mock(EventLogger.class);
diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeTestHelper.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeTestHelper.java
index 0fd06645bc..ac9ad8251a 100644
--- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeTestHelper.java
+++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeTestHelper.java
@@ -45,6 +45,9 @@ import org.apache.qpid.server.model.ConfiguredObjectFactory;
 import org.apache.qpid.server.model.RemoteReplicationNode;
 import org.apache.qpid.server.model.State;
 import org.apache.qpid.server.model.VirtualHostNode;
+import org.apache.qpid.server.store.ConfiguredObjectRecord;
+import org.apache.qpid.server.store.ConfiguredObjectRecordImpl;
+import org.apache.qpid.server.store.UnresolvedConfiguredObject;
 import org.apache.qpid.server.util.BrokerTestHelper;
 import org.apache.qpid.test.utils.QpidTestCase;
 import org.apache.qpid.util.FileUtils;
@@ -194,6 +197,20 @@ public class BDBHAVirtualHostNodeTestHelper
         return node;
     }
 
+    public BDBHAVirtualHostNode<?> recoverHaVHN(UUID id, Map<String, Object> attributes)
+    {
+        Map<String,ConfiguredObjectRecord> parents = new HashMap<>();
+        parents.put(Broker.class.getSimpleName(),new ConfiguredObjectRecordImpl(_broker.getId(), Broker.class.getSimpleName(), Collections.<String, Object>singletonMap(Broker.NAME, _broker.getName()) ));
+        ConfiguredObjectRecordImpl record = new ConfiguredObjectRecordImpl(id, VirtualHostNode.class.getSimpleName(), attributes, parents );
+
+        @SuppressWarnings("unchecked")
+        UnresolvedConfiguredObject<BDBHAVirtualHostNodeImpl> unresolved =  _objectFactory.recover(record, _broker);
+        BDBHAVirtualHostNode<?> node = unresolved.resolve();
+        node.open();
+        _nodes.add(node);
+        return node;
+    }
+
     public void assertNodeRole(BDBHAVirtualHostNode<?> node, String... roleName) throws InterruptedException
     {
         int iterationCounter = 0;