QPID-5715: On BDB HA virtual host node deletion invoke ReplicationGroupAdmin#removeMember to remove node from the group

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1596273 13f79535-47bb-0310-9956-ffa450edef68

6 files changed, 245 insertions, 9 deletions

diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
index cde00a8804..91e73324bc 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
@@ -23,12 +23,10 @@ package org.apache.qpid.server.store.berkeleydb.replication;
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
@@ -75,6 +73,7 @@ import com.sleepycat.je.rep.StateChangeEvent;
 import com.sleepycat.je.rep.StateChangeListener;
 import com.sleepycat.je.rep.util.DbPing;
 import com.sleepycat.je.rep.util.ReplicationGroupAdmin;
+import com.sleepycat.je.rep.utilint.HostPortPair;
 import com.sleepycat.je.rep.utilint.ServiceDispatcher.ServiceConnectFailedException;
 import com.sleepycat.je.rep.vlsn.VLSNRange;
 import com.sleepycat.je.utilint.PropUtil;
@@ -720,7 +719,7 @@ public class ReplicatedEnvironmentFacade implements EnvironmentFacade, StateChan
         helpers.addAll(_environment.getRepConfig().getHelperSockets());
 
         final ReplicationConfig repConfig = _environment.getRepConfig();
-        helpers.add(InetSocketAddress.createUnresolved(repConfig.getNodeHostname(), repConfig.getNodePort()));
+        helpers.add(HostPortPair.getSocket(HostPortPair.getString(repConfig.getNodeHostname(), repConfig.getNodePort())));
 
         return new ReplicationGroupAdmin(_configuration.getGroupName(), helpers);
     }
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
index d1e6b39bcc..20c80ad765 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
@@ -21,7 +21,6 @@
 
 package org.apache.qpid.server.virtualhostnode.berkeleydb;
 
-import java.io.File;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
@@ -98,6 +97,12 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         return _lastTransactionId;
     }
 
+    @Override
+    public void deleted()
+    {
+        super.deleted();
+    }
+
     @StateTransition(currentState = {State.ACTIVE, State.STOPPED}, desiredState = State.DELETED)
     private void doDelete()
     {
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
index 9602c2c29c..c019465176 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
@@ -20,9 +20,11 @@
  */
 package org.apache.qpid.server.virtualhostnode.berkeleydb;
 
+import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
@@ -32,11 +34,14 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import javax.security.auth.Subject;
 
+import com.sleepycat.je.DatabaseException;
 import com.sleepycat.je.rep.NodeState;
 import com.sleepycat.je.rep.ReplicatedEnvironment;
 import com.sleepycat.je.rep.ReplicationNode;
 import com.sleepycat.je.rep.StateChangeEvent;
 import com.sleepycat.je.rep.StateChangeListener;
+import com.sleepycat.je.rep.util.ReplicationGroupAdmin;
+import com.sleepycat.je.rep.utilint.HostPortPair;
 
 import org.apache.log4j.Logger;
 import org.apache.qpid.server.logging.messages.ConfigStoreMessages;
@@ -340,6 +345,39 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
         }
     }
 
+    @StateTransition( currentState = { State.ACTIVE, State.STOPPED, State.ERRORED}, desiredState = State.DELETED )
+    protected void doDelete()
+    {
+        Set<InetSocketAddress> helpers = getRemoteNodeAddresses();
+        super.doDelete();
+        if (getState() == State.DELETED && !helpers.isEmpty())
+        {
+            try
+            {
+                new ReplicationGroupAdmin(_groupName, helpers).removeMember(getName());
+            }
+            catch(DatabaseException e)
+            {
+                LOGGER.warn("The deletion of node " + this + " on remote nodes failed due to: " + e.getMessage()
+                        + ". To finish deletion a removal of the node from any of remote nodes (" + helpers + ") is required.");
+            }
+        }
+    }
+
+    private Set<InetSocketAddress> getRemoteNodeAddresses()
+    {
+        Set<InetSocketAddress> helpers = new HashSet<InetSocketAddress>();
+        @SuppressWarnings("rawtypes")
+        Collection<? extends RemoteReplicationNode> remoteNodes = getRemoteReplicationNodes();
+        for (RemoteReplicationNode<?> node : remoteNodes)
+        {
+            BDBHARemoteReplicationNode<?> bdbHaRemoteReplicationNode = (BDBHARemoteReplicationNode<?>)node;
+            String remoteNodeAddress = bdbHaRemoteReplicationNode.getAddress();
+            helpers.add(HostPortPair.getSocket(remoteNodeAddress));
+        }
+        return helpers;
+    }
+
     protected void onClose()
     {
         try
@@ -626,8 +664,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             BDBHARemoteReplicationNodeImpl remoteNode = getChildByName(BDBHARemoteReplicationNodeImpl.class, node.getName());
             if (remoteNode != null)
             {
-                remoteNode.delete();
-                childRemoved(remoteNode);
+                remoteNode.deleted();
             }
         }
 
diff --git a/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js b/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js
index c4ac3f3d91..9dbebc9d7d 100644
--- a/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js
+++ b/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js
@@ -148,7 +148,7 @@ define(["dojo/_base/xhr",
       {
         var data = that.membersGrid.grid.selection.getSelected();
         that.transferMasterButton.set("disabled", data.length != 1|| data[0].role != "REPLICA");
-        that.removeNodeButton.set("disabled", data.length != 1 || data[0].role == "MASTER" || data[0].name ==  that.data.name);
+        that.removeNodeButton.set("disabled", data.length != 1 || data[0].role == "MASTER");
       };
       connect.connect(this.membersGrid.grid.selection, 'onSelected',  nodeControlsToggler);
       connect.connect(this.membersGrid.grid.selection, 'onDeselected',  nodeControlsToggler);
@@ -170,8 +170,14 @@ define(["dojo/_base/xhr",
             var data = that.membersGrid.grid.selection.getSelected();
             if (data.length == 1 && confirm("Are you sure you would like to delete node '" + data[0].name + "'?"))
             {
-                sendRequest(that.data.name, data[0].name, "DELETE");
-                that.membersGrid.grid.selection.clear();
+                if (sendRequest(that.data.name, data[0].name, "DELETE"))
+                {
+                  that.membersGrid.grid.selection.clear();
+                  if (data[0].name == that.data.name)
+                  {
+                    that.parent.destroy();
+                  }
+                }
             }
           }
       );
diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeTest.java
index 6fd7b0bc1d..eb7cc03387 100644
--- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeTest.java
+++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeTest.java
@@ -471,6 +471,129 @@ public class BDBHAVirtualHostNodeTest extends QpidTestCase
     }
 
 
+    public void testRemoveReplicaNode() throws Exception
+    {
+        int node1PortNumber = findFreePort();
+        String helperAddress = "localhost:" + node1PortNumber;
+        String groupName = "group";
+
+        Map<String, Object> node1Attributes = new HashMap<String, Object>();
+        node1Attributes.put(BDBHAVirtualHostNode.ID, UUID.randomUUID());
+        node1Attributes.put(BDBHAVirtualHostNode.TYPE, "BDB_HA");
+        node1Attributes.put(BDBHAVirtualHostNode.NAME, "node1");
+        node1Attributes.put(BDBHAVirtualHostNode.GROUP_NAME, groupName);
+        node1Attributes.put(BDBHAVirtualHostNode.ADDRESS, helperAddress);
+        node1Attributes.put(BDBHAVirtualHostNode.HELPER_ADDRESS, helperAddress);
+        node1Attributes.put(BDBHAVirtualHostNode.STORE_PATH, _bdbStorePath + File.separator + "1");
+
+        BDBHAVirtualHostNode<?> node1 = createHaVHN(node1Attributes);
+        node1.start();
+        assertEquals("Failed to activate node", State.ACTIVE, node1.getState());
+
+        int node2PortNumber = getNextAvailable(node1PortNumber+1);
+
+        Map<String, Object> node2Attributes = new HashMap<String, Object>();
+        node2Attributes.put(BDBHAVirtualHostNode.ID, UUID.randomUUID());
+        node2Attributes.put(BDBHAVirtualHostNode.TYPE, "BDB_HA");
+        node2Attributes.put(BDBHAVirtualHostNode.NAME, "node2");
+        node2Attributes.put(BDBHAVirtualHostNode.GROUP_NAME, groupName);
+        node2Attributes.put(BDBHAVirtualHostNode.ADDRESS, "localhost:" + node2PortNumber);
+        node2Attributes.put(BDBHAVirtualHostNode.HELPER_ADDRESS, helperAddress);
+        node2Attributes.put(BDBHAVirtualHostNode.STORE_PATH, _bdbStorePath + File.separator + "2");
+
+        BDBHAVirtualHostNode<?> node2 = createHaVHN(node2Attributes);
+        node2.start();
+        assertEquals("Failed to activate node2", State.ACTIVE, node2.getState());
+
+        int node3PortNumber = getNextAvailable(node2PortNumber+1);
+        Map<String, Object> node3Attributes = new HashMap<String, Object>();
+        node3Attributes.put(BDBHAVirtualHostNode.ID, UUID.randomUUID());
+        node3Attributes.put(BDBHAVirtualHostNode.TYPE, "BDB_HA");
+        node3Attributes.put(BDBHAVirtualHostNode.NAME, "node3");
+        node3Attributes.put(BDBHAVirtualHostNode.GROUP_NAME, groupName);
+        node3Attributes.put(BDBHAVirtualHostNode.ADDRESS, "localhost:" + node3PortNumber);
+        node3Attributes.put(BDBHAVirtualHostNode.HELPER_ADDRESS, helperAddress);
+        node3Attributes.put(BDBHAVirtualHostNode.STORE_PATH, _bdbStorePath + File.separator + "3");
+        BDBHAVirtualHostNode<?> node3 = createHaVHN(node3Attributes);
+        node3.start();
+        assertEquals("Failed to activate node3", State.ACTIVE, node3.getState());
+
+        BDBHAVirtualHostNode<?> master = awaitAndFindNodeInRole("MASTER");
+        awaitRemoteNodes(master, 2);
+
+        BDBHAVirtualHostNode<?> replica = awaitAndFindNodeInRole("REPLICA");
+
+        assertNotNull("Remote node " + replica.getName() + " is not found", findRemoteNode( master, replica.getName()));
+        replica.delete();
+
+        awaitRemoteNodes(master, 1);
+
+        assertNull("Remote node " + replica.getName() + " is not found", findRemoteNode( master, replica.getName()));
+    }
+
+    private BDBHARemoteReplicationNode<?> findRemoteNode(BDBHAVirtualHostNode<?> node, String name)
+    {
+        for (RemoteReplicationNode<?> remoteNode : node.getRemoteReplicationNodes())
+        {
+            if (remoteNode.getName().equals(name))
+            {
+                return (BDBHARemoteReplicationNode<?>)remoteNode;
+            }
+        }
+        return null;
+    }
+
+    private void awaitRemoteNodes(BDBHAVirtualHostNode<?> node, int expectedNodeNumber) throws InterruptedException
+    {
+        int counter = 0;
+
+        @SuppressWarnings("rawtypes")
+        Collection<? extends RemoteReplicationNode> remoteNodes = null;
+        do
+        {
+            remoteNodes = node.getRemoteReplicationNodes();
+            if (counter > 0)
+            {
+                Thread.sleep(100);
+            }
+            counter++;
+        }
+        while(remoteNodes.size() != expectedNodeNumber && counter<50);
+        assertEquals("Unexpected node number", expectedNodeNumber, node.getRemoteReplicationNodes().size());
+    }
+
+    private BDBHAVirtualHostNode<?> awaitAndFindNodeInRole(String role) throws InterruptedException
+    {
+        BDBHAVirtualHostNode<?> replica = null;
+        int findReplicaCount = 0;
+        while(replica == null)
+        {
+            replica = findNodeInRole(role);
+            if (replica == null)
+            {
+                Thread.sleep(100);
+            }
+            if (findReplicaCount > 50)
+            {
+                fail("Could not find a node in replica role");
+            }
+            findReplicaCount++;
+        }
+        return replica;
+    }
+
+    private BDBHAVirtualHostNode<?> findNodeInRole(String role)
+    {
+        for (BDBHAVirtualHostNode<?> node : _nodes)
+        {
+            if (role.equals(node.getRole()))
+            {
+                return node;
+            }
+        }
+        return null;
+    }
+
     private BDBHAVirtualHostNode<?> createHaVHN(Map<String, Object> attributes)
     {
         BDBHAVirtualHostNode<?> node = (BDBHAVirtualHostNode<?>) _objectFactory.create(VirtualHostNode.class, attributes, _broker);
diff --git a/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeRestTest.java b/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeRestTest.java
index 06e01aadc9..507ff07ee5 100644
--- a/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeRestTest.java
+++ b/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHostNodeRestTest.java
@@ -24,6 +24,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -107,6 +108,71 @@ public class BDBHAVirtualHostNodeRestTest extends QpidRestTestCase
         assertRemoteNodes(NODE1, NODE2, NODE3);
     }
 
+    public void testDeleteReplicaNode() throws Exception
+    {
+        createHANode(NODE1, _node1HaPort, _node1HaPort);
+        createHANode(NODE2, _node2HaPort, _node1HaPort);
+        createHANode(NODE3, _node3HaPort, _node1HaPort);
+
+        assertRemoteNodes(NODE1, NODE2, NODE3);
+
+        List<Map<String,Object>> data = getRestTestHelper().getJsonAsList("replicationnode/" + NODE1);
+        assertEquals("Unexpected number of remote nodes on " + NODE1, 2, data.size());
+
+        int responseCode = getRestTestHelper().submitRequest(_baseNodeRestUrl + NODE2, "DELETE");
+        assertEquals("Unexpected response code on deletion of virtual host node " + NODE2, 200, responseCode);
+
+        int counter = 0;
+        while (data.size() != 1 && counter<50)
+        {
+            data = getRestTestHelper().getJsonAsList("replicationnode/" + NODE1);
+            if (data.size() != 1)
+            {
+                Thread.sleep(100l);
+            }
+        }
+        assertEquals("Unexpected number of remote nodes on " + NODE1, 1, data.size());
+    }
+
+    public void testDeleteMasterNode() throws Exception
+    {
+        createHANode(NODE1, _node1HaPort, _node1HaPort);
+        createHANode(NODE2, _node2HaPort, _node1HaPort);
+        createHANode(NODE3, _node3HaPort, _node1HaPort);
+
+        assertNode(NODE1, _node1HaPort, _node1HaPort, NODE1);
+        assertRemoteNodes(NODE1, NODE2, NODE3);
+
+        // change priority to make Node2 a master
+        int responseCode = getRestTestHelper().submitRequest(_baseNodeRestUrl + NODE2, "PUT", Collections.<String,Object>singletonMap(BDBHAVirtualHostNode.PRIORITY, 100));
+        assertEquals("Unexpected response code on priority update of virtual host node " + NODE2, 200, responseCode);
+
+        List<Map<String,Object>> data = getRestTestHelper().getJsonAsList("replicationnode/" + NODE2);
+        assertEquals("Unexpected number of remote nodes on " + NODE2, 2, data.size());
+
+        // delete master
+        responseCode = getRestTestHelper().submitRequest(_baseNodeRestUrl + NODE1, "DELETE");
+        assertEquals("Unexpected response code on deletion of virtual host node " + NODE1, 200, responseCode);
+
+        // wait for new master
+        waitForAttributeChanged(_baseNodeRestUrl + NODE2 + "?depth=0", BDBHAVirtualHostNode.ROLE, "MASTER");
+
+        // delete remote node
+        responseCode = getRestTestHelper().submitRequest("replicationnode/" + NODE2 + "/" + NODE1, "DELETE");
+        assertEquals("Unexpected response code on deletion of remote node " + NODE1, 200, responseCode);
+
+        int counter = 0;
+        while (data.size() != 1 && counter<50)
+        {
+            data = getRestTestHelper().getJsonAsList("replicationnode/" + NODE2);
+            if (data.size() != 1)
+            {
+                Thread.sleep(100l);
+            }
+        }
+        assertEquals("Unexpected number of remote nodes on " + NODE2, 1, data.size());
+    }
+
     private void createHANode(String nodeName, int nodePort, int helperPort) throws IOException, JsonGenerationException, JsonMappingException
     {
         Map<String, Object> nodeData = new HashMap<String, Object>();