QPID-5513: HA backup fails if number of replicated queues exceeds number of channels.

The problem:
- create cluster of 2 brokers.
- create more than 32768 queues (exceeds number of channels on a connection)
- backup exits with critical error but
- client creating queues receives no error, primary continues with unreplicated queue.

The solution: Primary raises an error to the client if it attempts to create
queues in excess of the channel limit. The queue is not created on primary
or backup, primary and backup continue as normal.

In addition: raised the channel limit from 32k to 64k. There was no reason for
the smaller limit. See discussion: http://qpid.2158936.n2.nabble.com/CHANNEL-MAX-and-CHANNEL-HIGH-BIT-question-tp7603121p7603138.html

New unit test to reproduce the issue, must create > 64k queues.

Other minor improvements:
- brokertest framework doesn't override --log options in the arguments.
- increased default heartbeat in test framework for tests that have busy brokers.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1561206 13f79535-47bb-0310-9956-ffa450edef68

5 files changed, 92 insertions, 7 deletions

diff --git a/qpid/cpp/src/tests/CMakeLists.txt b/qpid/cpp/src/tests/CMakeLists.txt
index a4093c82a2..abcb82b97d 100644
--- a/qpid/cpp/src/tests/CMakeLists.txt
+++ b/qpid/cpp/src/tests/CMakeLists.txt
@@ -299,6 +299,10 @@ add_executable (msg_group_test msg_group_test.cpp ${platform_test_additions})
 target_link_libraries (msg_group_test qpidmessaging qpidtypes qpidcommon "${Boost_PROGRAM_OPTIONS_LIBRARY}")
 remember_location(msg_group_test)
 
+add_executable (ha_test_max_queues ha_test_max_queues.cpp ${platform_test_additions})
+target_link_libraries (ha_test_max_queues qpidmessaging qpidtypes qpidcommon "${Boost_PROGRAM_OPTIONS_LIBRARY}")
+remember_location(ha_test_max_queues)
+
 if (BUILD_SASL)
     add_executable (sasl_version sasl_version.cpp ${platform_test_additions})
     remember_location(sasl_version)
diff --git a/qpid/cpp/src/tests/brokertest.py b/qpid/cpp/src/tests/brokertest.py
index 3f99d8647b..c7fcf2c3af 100644
--- a/qpid/cpp/src/tests/brokertest.py
+++ b/qpid/cpp/src/tests/brokertest.py
@@ -247,7 +247,7 @@ class Broker(Popen):
     def get_log(self):
         return os.path.abspath(self.log)
 
-    def __init__(self, test, args=[], test_store=False, name=None, expect=EXPECT_RUNNING, port=0, log_level=None, wait=None, show_cmd=False):
+    def __init__(self, test, args=[], test_store=False, name=None, expect=EXPECT_RUNNING, port=0, wait=None, show_cmd=False):
         """Start a broker daemon. name determines the data-dir and log
         file names."""
 
@@ -276,7 +276,10 @@ class Broker(Popen):
 
         cmd += ["--log-to-file", self.log]
         cmd += ["--log-to-stderr=no"]
-        cmd += ["--log-enable=%s"%(log_level or "info+") ]
+
+        # Add default --log-enable arguments unless args already has --log arguments.
+        if not next((l for l in args if l.startswith("--log")), None):
+            args += ["--log-enable=info+"]
 
         if test_store: cmd += ["--load-module", BrokerTest.test_store_lib,
                                "--test-store-events", self.store_log]
@@ -474,9 +477,9 @@ class BrokerTest(TestCase):
         self.cleanup_stop(p)
         return p
 
-    def broker(self, args=[], name=None, expect=EXPECT_RUNNING, wait=True, port=0, log_level=None, show_cmd=False):
+    def broker(self, args=[], name=None, expect=EXPECT_RUNNING, wait=True, port=0, show_cmd=False):
         """Create and return a broker ready for use"""
-        b = Broker(self, args=args, name=name, expect=expect, port=port, log_level=log_level, show_cmd=show_cmd)
+        b = Broker(self, args=args, name=name, expect=expect, port=port, show_cmd=show_cmd)
         if (wait):
             try: b.ready()
             except Exception, e:
diff --git a/qpid/cpp/src/tests/ha_test.py b/qpid/cpp/src/tests/ha_test.py
index 8dd1347144..4606fab746 100755
--- a/qpid/cpp/src/tests/ha_test.py
+++ b/qpid/cpp/src/tests/ha_test.py
@@ -121,7 +121,7 @@ class HaBroker(Broker):
     @param client_cred: (user, password, mechanism) for admin clients started by the HaBroker.
     """
 
-    heartbeat=2
+    heartbeat=5
 
     def __init__(self, test, ha_port=None, args=[], brokers_url=None, ha_cluster=True,
                  ha_replicate="all", client_credentials=None, **kwargs):
@@ -129,8 +129,6 @@ class HaBroker(Broker):
         ha_port = ha_port or HaPort(test)
         args = copy(args)
         args += ["--load-module", BrokerTest.ha_lib,
-                 "--log-enable=info+",
-                 "--log-enable=debug+:ha::",
                  # Non-standard settings for faster tests.
                  "--link-maintenance-interval=0.1",
                  # Heartbeat and negotiate time are needed so that a broker wont
@@ -138,6 +136,9 @@ class HaBroker(Broker):
                  "--link-heartbeat-interval=%s"%(HaBroker.heartbeat),
                  "--max-negotiate-time=1000",
                  "--ha-cluster=%s"%ha_cluster]
+        # Add default --log-enable arguments unless args already has --log arguments.
+        if not next((l for l in args if l.startswith("--log")), None):
+            args += ["--log-enable=info+", "--log-enable=debug+:ha::"]
         if ha_replicate is not None:
             args += [ "--ha-replicate=%s"%ha_replicate ]
         if brokers_url: args += [ "--ha-brokers-url", brokers_url ]
diff --git a/qpid/cpp/src/tests/ha_test_max_queues.cpp b/qpid/cpp/src/tests/ha_test_max_queues.cpp
new file mode 100644
index 0000000000..fda271f854
--- /dev/null
+++ b/qpid/cpp/src/tests/ha_test_max_queues.cpp
@@ -0,0 +1,64 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+#include <qpid/client/Connection.h>
+#include <qpid/client/Session.h>
+#include <qpid/client/AsyncSession.h>
+#include <qpid/Url.h>
+#include <qpid/framing/reply_exceptions.h>
+#include <sstream>
+
+using namespace qpid::client;
+using namespace std;
+
+int main(int argc, char** argv) {
+    assert(argc == 2);          // Expecing URL of broker as argv[1]
+    try {
+        // We need to create a large number of queues quickly, so we
+        // use the old API for it's asynchronous commands.
+        // The qpid::messaging API does not allow async queue creation.
+        //
+        Connection c;
+        c.open(qpid::Url(argv[1]));
+        AsyncSession s = async(c.newSession());
+        // Generate too many queues, make sure we get an exception.
+        for (uint64_t i = 0; i < 100000; ++i) {
+            ostringstream os;
+            os << "q" << i;
+            string q = os.str();
+            s.queueDeclare(q, arg::sync=false);
+            if (i && i % 1000 == 0) {
+                s.sync();       // Check for exceptions.
+                cout << "Declared " << q << endl;
+            }
+        }
+        cout << "Expected resource-limit-exceeded exception" << endl;
+        return 1;
+    }
+    catch (const qpid::framing::ResourceLimitExceededException& e) {
+        cout << "Resource limit exceeded: " << e.what() << endl;
+        return 0;
+    }
+    catch (const std::exception& e) {
+        cout << "Error: " << e.what() << endl;
+        return 1;
+    }
+}
diff --git a/qpid/cpp/src/tests/ha_tests.py b/qpid/cpp/src/tests/ha_tests.py
index b8644ab0fa..abc62b643e 100755
--- a/qpid/cpp/src/tests/ha_tests.py
+++ b/qpid/cpp/src/tests/ha_tests.py
@@ -22,6 +22,7 @@ import os, signal, sys, time, imp, re, subprocess, glob, random, logging, shutil
 import traceback
 from qpid.messaging import Message, SessionError, NotFound, ConnectionError, ReceiverError, Connection, Timeout, Disposition, REJECTED, Empty, ServerError
 from qpid.datatypes import uuid4, UUID
+from qpid.harness import Skipped
 from brokertest import *
 from ha_test import *
 from threading import Thread, Lock, Condition
@@ -1078,6 +1079,18 @@ class LongTests(HaBrokerTest):
         finally:
             for t in threads: t.stopped = True; t.join()
 
+    def test_max_queues(self):
+        """Verify that we behave properly if we try to exceed the max number
+        of replicated queues - currently limited by the max number of channels
+        in the replication link"""
+        # This test is very slow (3 mins), skip it unless duration() > 1 minute.
+        if self.duration() < 60: return
+        # This test is written in C++ for speed, it takes a long time
+        # to create 64k queues in python. See ha_test_max_queues.cpp.
+        cluster = HaCluster(self, 2)
+        test = self.popen(["ha_test_max_queues", cluster[0].host_port()])
+        self.assertEqual(test.wait(), 0)
+
 class RecoveryTests(HaBrokerTest):
     """Tests for recovery after a failure."""