QPID-6222: Halt broker on running out of disk space

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1638052 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 58 insertions, 2 deletions

diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
index 9d3751caec..b0b3f61249 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
@@ -54,6 +54,7 @@ import com.sleepycat.je.Durability.SyncPolicy;
 import com.sleepycat.je.EnvironmentConfig;
 import com.sleepycat.je.EnvironmentFailureException;
 import com.sleepycat.je.ExceptionEvent;
+import com.sleepycat.je.LogWriteException;
 import com.sleepycat.je.Sequence;
 import com.sleepycat.je.SequenceConfig;
 import com.sleepycat.je.Transaction;
@@ -69,6 +70,7 @@ import com.sleepycat.je.utilint.PropUtil;
 import com.sleepycat.je.utilint.VLSN;
 import org.apache.log4j.Logger;
 import org.apache.qpid.server.store.StoreException;
+import org.apache.qpid.server.util.ServerScopedRuntimeException;
 import org.codehaus.jackson.map.ObjectMapper;
 
 import org.apache.qpid.server.configuration.IllegalConfigurationException;
@@ -364,6 +366,13 @@ public class ReplicatedEnvironmentFacade implements EnvironmentFacade, StateChan
     @Override
     public RuntimeException handleDatabaseException(String contextMessage, final RuntimeException dbe)
     {
+        if (dbe instanceof LogWriteException)
+        {
+            // something wrong with the disk (for example, no space left on device)
+            // store cannot operate
+            throw new ServerScopedRuntimeException("Cannot save data into the store", dbe);
+        }
+
         if (dbe instanceof StoreException || dbe instanceof ConnectionScopedRuntimeException)
         {
             return dbe;
@@ -1468,6 +1477,22 @@ public class ReplicatedEnvironmentFacade implements EnvironmentFacade, StateChan
         }
     }
 
+    private void onException(final Exception e)
+    {
+        _groupChangeExecutor.submit(new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                ReplicationGroupListener listener = _replicationGroupListener.get();
+                if (listener != null)
+                {
+                    listener.onException(e);
+                }
+            }
+        });
+    }
+
     private class RemoteNodeStateLearner implements Callable<Void>
     {
         private Map<String, ReplicatedEnvironment.State> _previousGroupState = Collections.emptyMap();
@@ -1793,11 +1818,18 @@ public class ReplicatedEnvironmentFacade implements EnvironmentFacade, StateChan
         @Override
         public void exceptionThrown(final ExceptionEvent event)
         {
-            if (event.getException() instanceof RollbackException)
+            Exception exception = event.getException();
+
+            if (exception instanceof LogWriteException)
+            {
+                onException(exception);
+            }
+
+            if (exception instanceof RollbackException)
             {
                 // Usually caused use of weak durability options: node priority zero,
                 // designated primary, electable group override.
-                RollbackException re = (RollbackException) event.getException();
+                RollbackException re = (RollbackException) exception;
 
                 LOGGER.warn(_prettyGroupNodeName + " has transaction(s) ahead of the current master. These"
                             + " must be discarded to allow this node to rejoin the group."
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicationGroupListener.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicationGroupListener.java
index 896295708a..888be3adff 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicationGroupListener.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicationGroupListener.java
@@ -60,4 +60,10 @@ public interface ReplicationGroupListener
      * caused by use of the weak durability options such as node priority zero.
      */
     void onNodeRolledback();
+
+    /**
+     * Callback method to invoke on critical JE exceptions
+     * @param e je exception
+     */
+    void onException(Exception e);
 }
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
index 7d877a8b00..185c8c306b 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
@@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import javax.security.auth.Subject;
 
 import com.sleepycat.je.DatabaseException;
+import com.sleepycat.je.LogWriteException;
 import com.sleepycat.je.rep.NodeState;
 import com.sleepycat.je.rep.NodeType;
 import com.sleepycat.je.rep.ReplicatedEnvironment;
@@ -1125,6 +1126,18 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             getEventLogger().message(getVirtualHostNodeLogSubject(), HighAvailabilityMessages.NODE_ROLLEDBACK());
         }
 
+        @Override
+        public void onException(Exception e)
+        {
+            if (e instanceof LogWriteException)
+            {
+                // something wrong with the disk (for example, no space left on device)
+                // broker and store cannot operate
+                // TODO: VHN can be transitioned into ERRORED state
+                throw new ServerScopedRuntimeException("Cannot save data into the store", e);
+            }
+        }
+
         private Map<String, Object> nodeToAttributes(ReplicationNode replicationNode)
         {
             Map<String, Object> attributes = new HashMap<String, Object>();
diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacadeTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacadeTest.java
index ed468c0844..1f7a6ede94 100644
--- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacadeTest.java
+++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacadeTest.java
@@ -976,5 +976,10 @@ public class ReplicatedEnvironmentFacadeTest extends QpidTestCase
         {
         }
 
+        @Override
+        public void onException(Exception e)
+        {
+        }
+
     }
 }