Merge branch 'master' into rabbitmq-server-591

16 files changed, 141 insertions, 45 deletions

diff --git a/.travis.yml b/.travis.yml
index 6d9dbc0b71..da19ebc302 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,6 @@
 sudo: false
+services:
+  - docker
 language: erlang
 notifications:
   email:
@@ -11,14 +13,20 @@ otp_release:
   - "R16B03-1"
   - "17.5"
   - "18.0"
+env:
+  matrix:
+  - GROUP=1 GROUP=2
 
 # The checkout made by Travis is a "detached HEAD". We switch back
 # to a tag or a branch. This pleases our git_rmq fetch method in
 # rabbitmq-components.mk and the proper tag/branch is selected in
 # dependencies too.
-before_script: (test "$TRAVIS_TAG" && git checkout "$TRAVIS_TAG") || (test "$TRAVIS_BRANCH" && git checkout "$TRAVIS_BRANCH")
+before_script:
+  - ([ "${GROUP}" = "1" ] && ((test "$TRAVIS_TAG" && git checkout "$TRAVIS_TAG") || (test "$TRAVIS_BRANCH" && git checkout "$TRAVIS_BRANCH")) || /bin/true)
 
-script: travis_wait make tests
+script:
+  - ([ "${GROUP}" = "1" ] && travis_wait make tests)
+  - ([ "${GROUP}" = "2" ] && sh ./scripts/travis_test_ocf_ra.sh)
 
 cache:
   apt: true
diff --git a/docs/README-for-packages b/docs/README-for-packages
index 35a1523ac3..f26889b640 100644
--- a/docs/README-for-packages
+++ b/docs/README-for-packages
@@ -18,3 +18,7 @@ An example configuration file is provided in the same directory as
 this README. Copy it to /etc/rabbitmq/rabbitmq.config to use it. The
 RabbitMQ server must be restarted after changing the configuration
 file.
+
+An example policy file for HA queues is provided in the same directory
+as this README. Copy and chmod +x it to
+/usr/local/sbin/set_rabbitmq_policy to use it with the Pacemaker OCF RA.
diff --git a/docs/rabbitmq.conf.example b/docs/rabbitmq.conf.example
index 5c92165f73..7e60c53da4 100644
--- a/docs/rabbitmq.conf.example
+++ b/docs/rabbitmq.conf.example
@@ -402,13 +402,13 @@
 ##
 
 # QA: Maybe use IP type like in tcp_listener?
-# management.listeners.port = 12345
-# management.listeners.ip   = 127.0.0.1
-# management.listeners.ssl  = true
+# management.listener.port = 12345
+# management.listener.ip   = 127.0.0.1
+# management.listener.ssl  = true
 
-# management.listeners.ssl_opts.cacertfile = /path/to/cacert.pem
-# management.listeners.ssl_opts.certfile   = /path/to/cert.pem
-# management.listeners.ssl_opts.keyfile    = /path/to/key.pem
+# management.listener.ssl_opts.cacertfile = /path/to/cacert.pem
+# management.listener.ssl_opts.certfile   = /path/to/cert.pem
+# management.listener.ssl_opts.keyfile    = /path/to/key.pem
 
 ## One of 'basic', 'detailed' or 'none'. See
 ## http://www.rabbitmq.com/management.html#fine-stats for more details.
diff --git a/scripts/set_rabbitmq_policy.sh b/docs/set_rabbitmq_policy.sh.example
index a88b0c417a..f46e901ad5 100644
--- a/scripts/set_rabbitmq_policy.sh
+++ b/docs/set_rabbitmq_policy.sh.example
@@ -2,4 +2,3 @@
 # cluster start up. It is a convenient place to set your cluster
 # policy here, for example:
 # ${OCF_RESKEY_ctl} set_policy ha-all "." '{"ha-mode":"all", "ha-sync-mode":"automatic"}' --apply-to all --priority 0
-
diff --git a/packaging/RPMS/Fedora/rabbitmq-server.spec b/packaging/RPMS/Fedora/rabbitmq-server.spec
index d3d33aa985..b6715d430a 100644
--- a/packaging/RPMS/Fedora/rabbitmq-server.spec
+++ b/packaging/RPMS/Fedora/rabbitmq-server.spec
@@ -26,7 +26,6 @@ RabbitMQ is an open source multi-protocol messaging broker.
 %define _rabbit_server_ocf scripts/rabbitmq-server.ocf
 %define _plugins_state_dir %{_localstatedir}/lib/rabbitmq/plugins
 %define _rabbit_server_ha_ocf scripts/rabbitmq-server-ha.ocf
-%define _set_rabbitmq_policy_sh scripts/set_rabbitmq_policy.sh
 
 
 %define _maindir %{buildroot}%{_rabbit_erllibdir}
@@ -51,7 +50,6 @@ mkdir -p %{buildroot}%{_localstatedir}/log/rabbitmq
 install -p -D -m 0755 %{S:1} %{buildroot}%{_initrddir}/rabbitmq-server
 install -p -D -m 0755 %{_rabbit_server_ocf} %{buildroot}%{_exec_prefix}/lib/ocf/resource.d/rabbitmq/rabbitmq-server
 install -p -D -m 0755 %{_rabbit_server_ha_ocf} %{buildroot}%{_exec_prefix}/lib/ocf/resource.d/rabbitmq/rabbitmq-server-ha
-install -p -D -m 0644 %{_set_rabbitmq_policy_sh} %{buildroot}%{_exec_prefix}/lib/ocf/resource.d/rabbitmq/set_rabbitmq_policy.sh.example
 install -p -D -m 0644 %{S:2} %{buildroot}%{_sysconfdir}/logrotate.d/rabbitmq-server
 
 mkdir -p %{buildroot}%{_sysconfdir}/rabbitmq
@@ -127,6 +125,7 @@ done
 %doc LICENSE*
 %doc README
 %doc docs/rabbitmq.config.example
+%doc docs/set_rabbitmq_policy.sh.example
 
 %clean
 rm -rf %{buildroot}
diff --git a/packaging/debs/Debian/debian/rabbitmq-server.docs b/packaging/debs/Debian/debian/rabbitmq-server.docs
index 40d4f2dc81..fbe9f95a05 100644
--- a/packaging/debs/Debian/debian/rabbitmq-server.docs
+++ b/packaging/debs/Debian/debian/rabbitmq-server.docs
@@ -1 +1,2 @@
 docs/rabbitmq.config.example
+docs/set_rabbitmq_policy.sh.example
diff --git a/packaging/debs/Debian/debian/rules b/packaging/debs/Debian/debian/rules
index 669e30c177..053df18115 100755
--- a/packaging/debs/Debian/debian/rules
+++ b/packaging/debs/Debian/debian/rules
@@ -49,8 +49,6 @@ override_dh_auto_install:
 		$(DEB_DESTDIR)$(PREFIX)/lib/ocf/resource.d/rabbitmq/rabbitmq-server
 	install -p -D -m 0755 scripts/rabbitmq-server-ha.ocf \
 		$(DEB_DESTDIR)$(PREFIX)/lib/ocf/resource.d/rabbitmq/rabbitmq-server-ha
-	install -p -D -m 0644 scripts/set_rabbitmq_policy.sh \
-		$(DEB_DESTDIR)$(PREFIX)/lib/ocf/resource.d/rabbitmq/set_rabbitmq_policy.sh.example
 
 	rm $(DEB_DESTDIR)$(RMQ_ERLAPP_DIR)/LICENSE* \
 		$(DEB_DESTDIR)$(RMQ_ERLAPP_DIR)/INSTALL
diff --git a/scripts/rabbitmq-defaults b/scripts/rabbitmq-defaults
index bccd0d7435..1505ce0f00 100644..100755
--- a/scripts/rabbitmq-defaults
+++ b/scripts/rabbitmq-defaults
@@ -43,6 +43,5 @@ ADVANCED_CONFIG_FILE=${SYS_PREFIX}/etc/rabbitmq/advanced
 SCHEMA_DIR=${RABBITMQ_HOME}/priv/schema
 
 PLUGINS_DIR="${RABBITMQ_HOME}/plugins"
-IO_THREAD_POOL_SIZE=64
 
 CONF_ENV_FILE=${SYS_PREFIX}/etc/rabbitmq/rabbitmq-env.conf
diff --git a/scripts/rabbitmq-server b/scripts/rabbitmq-server
index 0e3f06baf9..98a4c70120 100755
--- a/scripts/rabbitmq-server
+++ b/scripts/rabbitmq-server
@@ -128,7 +128,23 @@ fi
 # there is no other way of preventing their expansion.
 set -f
 
+# Lazy initialization of threed pool size - if it wasn't set
+# explicitly. This parameter is only needed when server is starting,
+# so it makes no sense to do this calculations in rabbitmq-env or
+# rabbitmq-defaults scripts.
+ensure_thread_pool_size() {
+    if [ -z "${RABBITMQ_IO_THREAD_POOL_SIZE}" ]; then
+        RABBITMQ_IO_THREAD_POOL_SIZE=$(
+            ${ERL_DIR}erl -pa "$RABBITMQ_EBIN_ROOT" \
+                      -boot "${CLEAN_BOOT_FILE}" \
+                      -noinput \
+                      -s rabbit_misc report_default_thread_pool_size
+        )
+    fi
+}
+
 start_rabbitmq_server() {
+    ensure_thread_pool_size
     check_start_params &&
     RABBITMQ_CONFIG_FILE=$RABBITMQ_CONFIG_FILE \
     exec ${ERL_DIR}erl \
diff --git a/scripts/rabbitmq-server-ha.ocf b/scripts/rabbitmq-server-ha.ocf
index c6d0e26241..5ead9a81ad 100755
--- a/scripts/rabbitmq-server-ha.ocf
+++ b/scripts/rabbitmq-server-ha.ocf
@@ -13,8 +13,8 @@
 #
 # See usage() function below for more details ...
 #
-# Note that the script uses set_rabbitmq_policy.sh script located in the
-# same directory to setup RabbitMQ policies.
+# Note that the script uses an external file to setup RabbitMQ policies
+# so make sure to create it from an example shipped with the package.
 #
 #######################################################################
 # Initialization:
@@ -46,6 +46,7 @@ OCF_RESKEY_erlang_cookie_file_default="/var/lib/rabbitmq/.erlang.cookie"
 OCF_RESKEY_use_fqdn_default=false
 OCF_RESKEY_fqdn_prefix_default=""
 OCF_RESKEY_max_rabbitmqctl_timeouts_default=3
+OCF_RESKEY_policy_file_default="/usr/local/sbin/set_rabbitmq_policy"
 
 : ${HA_LOGTAG="lrmd"}
 : ${HA_LOGFACILITY="daemon"}
@@ -66,6 +67,7 @@ OCF_RESKEY_max_rabbitmqctl_timeouts_default=3
 : ${OCF_RESKEY_use_fqdn=${OCF_RESKEY_use_fqdn_default}}
 : ${OCF_RESKEY_fqdn_prefix=${OCF_RESKEY_fqdn_prefix_default}}
 : ${OCF_RESKEY_max_rabbitmqctl_timeouts=${OCF_RESKEY_max_rabbitmqctl_timeouts_default}}
+: ${OCF_RESKEY_policy_file=${OCF_RESKEY_policy_file_default}}
 
 #######################################################################
 
@@ -288,6 +290,14 @@ If too many timeouts happen in a raw, the monitor call will return with error.
 <content type="string" default="${OCF_RESKEY_max_rabbitmqctl_timeouts_default}" />
 </parameter>
 
+<parameter name="policy_file" unique="0" required="0">
+<longdesc lang="en">
+A path to the shell script to setup RabbitMQ policies
+</longdesc>
+<shortdesc lang="en">A policy file path</shortdesc>
+<content type="string" default="${OCF_RESKEY_policy_file_default}" />
+</parameter>
+
 $EXTENDED_OCF_PARAMS
 
 </parameters>
@@ -668,8 +678,8 @@ reset_mnesia() {
     # remove mnesia files, if required
     if $make_amnesia ; then
         kill_rmq_and_remove_pid
-        ocf_run rm -rf "${MNESIA_FILES}/*"
-        ocf_log warn "${LH} Mnesia files appear corrupted and have been removed."
+        ocf_run rm -rf "${MNESIA_FILES}"
+        ocf_log warn "${LH} Mnesia files appear corrupted and have been removed from ${MNESIA_FILES}."
     fi
     # always return OCF SUCCESS
     return $OCF_SUCCESS
@@ -2098,8 +2108,7 @@ action_promote() {
                         exit $OCF_FAILED_MASTER
                     fi
 
-                    local set_policy_path="$(dirname $0)/set_rabbitmq_policy.sh"
-                    [ -f $set_policy_path ] && . $set_policy_path
+                    [ -f "${OCF_RESKEY_policy_file}" ] && . "${OCF_RESKEY_policy_file}"
 
                     # create timestamp file
                     nowtime="$(now)"
diff --git a/scripts/rabbitmq-server.bat b/scripts/rabbitmq-server.bat
index 33e316a07f..2a38e77a82 100644
--- a/scripts/rabbitmq-server.bat
+++ b/scripts/rabbitmq-server.bat
@@ -109,10 +109,9 @@ if "!RABBITMQ_NODE_ONLY!"=="" (
 )

 

 if "!RABBITMQ_IO_THREAD_POOL_SIZE!"=="" (

-    set RABBITMQ_IO_THREAD_POOL_ARG=30

-) else (

-    set RABBITMQ_IO_THREAD_POOL_ARG=!RABBITMQ_IO_THREAD_POOL_SIZE!

-)

+    set RABBITMQ_IO_THREAD_POOL_SIZE=64

+) 

+

 

 set ENV_OK=true

 CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE! 

diff --git a/scripts/rabbitmq-service.bat b/scripts/rabbitmq-service.bat
index f302087f91..7e80e78398 100644
--- a/scripts/rabbitmq-service.bat
+++ b/scripts/rabbitmq-service.bat
@@ -106,7 +106,7 @@ if not exist "!RABBITMQ_BASE!" (
 )

 

 set ENV_OK=true

-CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE! 

+CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE!

 CALL :check_not_empty "RABBITMQ_NAME_TYPE" !RABBITMQ_NAME_TYPE!

 CALL :check_not_empty "RABBITMQ_NODENAME" !RABBITMQ_NODENAME!

 

@@ -150,10 +150,11 @@ if ERRORLEVEL 3 (
 

 if not exist "!RABBITMQ_SCHEMA_DIR!\rabbitmq.schema" (

     copy "!RABBITMQ_HOME!\priv\schema\rabbitmq.schema" "!RABBITMQ_SCHEMA_DIR!\rabbitmq.schema"

-    REM Try to create config file, if it doesn't exist
-    REM It still can fail to be created, but at least not for default install
-if not exist "!RABBITMQ_CONFIG_FILE!.config" (
-    echo []. > !RABBITMQ_CONFIG_FILE!.config
+)

+    REM Try to create advanced config file, if it doesn't exist

+    REM It still can fail to be created, but at least not for default install

+if not exist "!RABBITMQ_ADVANCED_CONFIG_FILE!.config" (

+    echo []. > !RABBITMQ_ADVANCED_CONFIG_FILE!.config

 )

 

 if exist "!RABBITMQ_CONFIG_FILE!.config" (

@@ -191,7 +192,7 @@ if "!RABBITMQ_NODE_ONLY!"=="" (
 )

 

 if "!RABBITMQ_IO_THREAD_POOL_SIZE!"=="" (

-    set RABBITMQ_IO_THREAD_POOL_SIZE=30

+    set RABBITMQ_IO_THREAD_POOL_SIZE=64

 )

 

 if "!RABBITMQ_SERVICE_RESTART!"=="" (

@@ -262,7 +263,7 @@ EXIT /B 0
 if "%~2"=="" (

     ECHO "Error: ENV variable should be defined: %1. Please check rabbitmq-env, rabbitmq-default, and !RABBITMQ_CONF_ENV_FILE! script files. Check also your Environment Variables settings"

     set ENV_OK=false

-    EXIT /B 78 

+    EXIT /B 78

     )

 EXIT /B 0

 

diff --git a/scripts/travis_test_ocf_ra.sh b/scripts/travis_test_ocf_ra.sh
new file mode 100644
index 0000000000..e8f9a74194
--- /dev/null
+++ b/scripts/travis_test_ocf_ra.sh
@@ -0,0 +1,30 @@
+#!/bin/sh -eux
+# Prepare and run a smoke test against the RabbitMQ OCF RA only if
+# the scripts/rabbitmq-server-ha.ocf has changes
+if ! git diff HEAD~ --name-only | grep -q scripts/rabbitmq-server-ha.ocf
+then
+  exit 0
+fi
+
+export VAGRANT_VERSION=1.8.1
+export DOCKER_IMAGE=bogdando/rabbitmq-cluster-ocf-wily
+export UPLOAD_METHOD=none
+export DOCKER_MOUNTS="$(pwd)/scripts/rabbitmq-server-ha.ocf:/tmp/rabbitmq-server-ha"
+
+# Install vagrant and requirements
+sudo apt-get install -qq git wget
+wget --no-verbose https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb
+sudo dpkg -i --force-all ./vagrant_${VAGRANT_VERSION}_x86_64.deb
+vagrant plugin install vagrant-triggers
+
+# Update docker and prepare images
+sudo apt-get update
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install --only-upgrade docker-engine
+sudo service docker restart
+docker pull $DOCKER_IMAGE
+
+# Prepare and run a smoke test for a rabbitmq cluster by the OCF RA
+git clone https://github.com/bogdando/rabbitmq-cluster-ocf-vagrant.git
+cd ./rabbitmq-cluster-ocf-vagrant
+vagrant up --provider docker
+docker exec -it n1 /bin/bash /vagrant/vagrant_script/test_rabbitcluster.sh rabbit@n1 rabbit@n2
diff --git a/src/rabbit_cli.erl b/src/rabbit_cli.erl
index 086f88d9f6..4151504956 100644
--- a/src/rabbit_cli.erl
+++ b/src/rabbit_cli.erl
@@ -18,17 +18,19 @@
 -include("rabbit_cli.hrl").
 
 -export([main/3, start_distribution/0, start_distribution/1,
-         parse_arguments/4, rpc_call/4, rpc_call/5, rpc_call/7]).
+         parse_arguments/4, filter_opts/2,
+         rpc_call/4, rpc_call/5, rpc_call/7]).
 
 %%----------------------------------------------------------------------------
 
 -ifdef(use_specs).
 
+-type(option_name() :: string()).
+-type(option_value() :: string() | node() | boolean()).
 -type(optdef() :: flag | {option, string()}).
--type(parse_result() :: {'ok', {atom(), [{string(), string()}], [string()]}} |
+-type(parse_result() :: {'ok', {atom(), [{option_name(), option_value()}], [string()]}} |
                         'no_command').
 
-
 -spec(main/3 :: (fun (([string()], string()) -> parse_result()),
                      fun ((atom(), atom(), [any()], [any()]) -> any()),
                          atom()) -> no_return()).
@@ -38,6 +40,9 @@
 -spec(parse_arguments/4 ::
         ([{atom(), [{string(), optdef()}]} | atom()],
          [{string(), optdef()}], string(), [string()]) -> parse_result()).
+
+-spec(filter_opts/2 :: ([{option_name(), option_value()}], [option_name()]) -> [boolean()]).
+
 -spec(rpc_call/4 :: (node(), atom(), atom(), [any()]) -> any()).
 -spec(rpc_call/5 :: (node(), atom(), atom(), [any()], number()) -> any()).
 -spec(rpc_call/7 :: (node(), atom(), atom(), [any()], reference(), pid(),
@@ -244,6 +249,22 @@ process_opts(Defs, C, [A | As], Found, KVs, Outs) ->
         {none, _, _}     -> no_command
     end.
 
+%% When we have a set of flags that are used for filtering, we want by
+%% default to include every such option in our output. But if a user
+%% explicitly specified any such flag, we want to include only items
+%% which he has requested.
+filter_opts(CurrentOptionValues, AllOptionNames) ->
+    Explicit = lists:map(fun(OptName) ->
+                                 proplists:get_bool(OptName, CurrentOptionValues)
+                         end,
+                         AllOptionNames),
+    case lists:member(true, Explicit) of
+        true ->
+            Explicit;
+        false ->
+            lists:duplicate(length(AllOptionNames), true)
+    end.
+
 %%----------------------------------------------------------------------------
 
 fmt_stderr(Format, Args) -> rabbit_misc:format_stderr(Format ++ "~n", Args).
diff --git a/src/rabbit_control_main.erl b/src/rabbit_control_main.erl
index b7a7b51242..6f99ec903c 100644
--- a/src/rabbit_control_main.erl
+++ b/src/rabbit_control_main.erl
@@ -73,7 +73,7 @@
          {clear_policy, [?VHOST_DEF]},
          {list_policies, [?VHOST_DEF]},
 
-         {list_queues, [?VHOST_DEF]},
+         {list_queues, [?VHOST_DEF, ?OFFLINE_DEF, ?ONLINE_DEF]},
          {list_exchanges, [?VHOST_DEF]},
          {list_bindings, [?VHOST_DEF]},
          {list_connections, [?VHOST_DEF]},
@@ -610,10 +610,11 @@ action(list_user_permissions, Node, Args = [_Username], _Opts, Inform, Timeout)
          true);
 
 action(list_queues, Node, Args, Opts, Inform, Timeout) ->
+    [Online, Offline] = rabbit_cli:filter_opts(Opts, [?ONLINE_OPT, ?OFFLINE_OPT]),
     Inform("Listing queues", []),
     VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)),
     ArgAtoms = default_if_empty(Args, [name, messages]),
-    call(Node, {rabbit_amqqueue, info_all, [VHostArg, ArgAtoms]},
+    call(Node, {rabbit_amqqueue, info_all, [VHostArg, ArgAtoms, Online, Offline]},
          ArgAtoms, Timeout);
 
 action(list_exchanges, Node, Args, Opts, Inform, Timeout) ->
@@ -750,15 +751,26 @@ default_if_empty(List, Default) when is_list(List) ->
        true       -> [list_to_atom(X) || X <- List]
     end.
 
+display_info_message_row(IsEscaped, Result, InfoItemKeys) ->
+    display_row([format_info_item(
+                   case proplists:lookup(X, Result) of
+                       none when is_list(Result), length(Result) > 0 ->
+                           exit({error, {bad_info_key, X}});
+                       none -> Result;
+                       {X, Value} -> Value
+                   end, IsEscaped) || X <- InfoItemKeys]).
+
 display_info_message(IsEscaped) ->
-    fun(Result, InfoItemKeys) ->
-            display_row([format_info_item(
-                           case proplists:lookup(X, Result) of
-                               none when is_list(Result), length(Result) > 0 ->
-                                   exit({error, {bad_info_key, X}});
-                               none -> Result;
-                               {X, Value} -> Value
-                           end, IsEscaped) || X <- InfoItemKeys])
+    fun ([], _) ->
+            ok;
+        ([FirstResult|_] = List, InfoItemKeys) when is_list(FirstResult) ->
+            lists:foreach(fun(Result) ->
+                                  display_info_message_row(IsEscaped, Result, InfoItemKeys)
+                          end,
+                          List),
+            ok;
+        (Result, InfoItemKeys) ->
+            display_info_message_row(IsEscaped, Result, InfoItemKeys)
     end.
 
 display_info_list(Results, InfoItemKeys) when is_list(Results) ->
diff --git a/src/rabbit_disk_monitor.erl b/src/rabbit_disk_monitor.erl
index 124306487e..88a8096fd4 100644
--- a/src/rabbit_disk_monitor.erl
+++ b/src/rabbit_disk_monitor.erl
@@ -235,7 +235,7 @@ parse_free_win32(CommandResult) ->
     list_to_integer(lists:reverse(Free)).
 
 interpret_limit({mem_relative, Relative}) 
-    when is_float(Relative), Relative < 1 ->
+    when is_float(Relative) ->
     round(Relative * vm_memory_monitor:get_total_memory());
 interpret_limit(Absolute) -> 
     case rabbit_resource_monitor_misc:parse_information_unit(Absolute) of