Merge branch 'stable'

Conflicts:
	docs/rabbitmq.config.example

3 files changed, 152 insertions, 51 deletions

diff --git a/docs/rabbitmq-server.service.example b/docs/rabbitmq-server.service.example
index 53a5c04aa7..898f8edfc2 100644
--- a/docs/rabbitmq-server.service.example
+++ b/docs/rabbitmq-server.service.example
@@ -21,6 +21,7 @@ TimeoutStartSec=3600
 WorkingDirectory=/var/lib/rabbitmq
 ExecStart=/usr/lib/rabbitmq/bin/rabbitmq-server
 ExecStop=/usr/lib/rabbitmq/bin/rabbitmqctl stop
+ExecStop=/bin/sh -c "while ps -p $MAINPID >/dev/null 2>&1; do sleep 1; done"
 
 [Install]
 WantedBy=multi-user.target
diff --git a/docs/rabbitmq.config.example b/docs/rabbitmq.config.example
index 77b7df743b..60af5b7eea 100644
--- a/docs/rabbitmq.config.example
+++ b/docs/rabbitmq.config.example
@@ -42,7 +42,7 @@
    %% connection. Hostnames will then be shown instead of IP addresses
    %% in rabbitmqctl and the management plugin.
    %%
-   %% {reverse_dns_lookups, true},
+   %% {reverse_dns_lookups, false},
 
    %%
    %% Security / AAA
@@ -114,12 +114,17 @@
    %%
    %% To use the SSL cert's CN instead of its DN as the username
    %%
-   %% {ssl_cert_login_from, common_name},
+   %% {ssl_cert_login_from, distinguished_name},
 
    %% SSL handshake timeout, in milliseconds.
    %%
    %% {ssl_handshake_timeout, 5000},
 
+   %% Makes RabbitMQ accept SSLv3 client connections by default.
+   %% DO NOT DO THIS IF YOU CAN HELP IT.
+   %%
+   %% {ssl_allow_poodle_attack, false},
+
    %% Password hashing implementation. Will only affect newly
    %% created users. To recalculate hash for an existing user
    %% it's necessary to update her password.
@@ -180,7 +185,10 @@
    %% =====================================================
    %%
 
-   %% Set the default AMQP heartbeat delay (in seconds).
+   %% Sets the default AMQP 0-9-1 heartbeat timeout in seconds.
+   %% Values lower than 6 can produce false positives and are not
+   %% recommended.
+   %% See http://www.rabbitmq.com/heartbeats.html.
    %%
    %% {heartbeat, 60},
 
@@ -196,12 +204,11 @@
    %% Set the max permissible number of channels per connection.
    %% 0 means "no limit".
    %%
-   %% {channel_max, 128},
+   %% {channel_max, 0},
 
-   %% Customising Socket Options.
+   %% TCP socket options.
    %%
-   %% See (http://www.erlang.org/doc/man/inet.html#setopts-2) for
-   %% further documentation.
+   %% See http://www.rabbitmq.com/networking.html.
    %%
    %% {tcp_listen_options, [{backlog,       128},
    %%                       {nodelay,       true},
@@ -255,7 +262,7 @@
    %% `rss` is the default. Introduced in 3.6.11.
    %% See https://github.com/rabbitmq/rabbitmq-server/issues/1223 for background.
    %% {vm_memory_calculation_strategy, rss},
-   
+
    %% Interval (in milliseconds) at which we perform the check of the memory
    %% levels against the watermarks.
    %%
@@ -287,8 +294,34 @@
    %% =====================
    %%
 
+   %% Queue master location strategy:
+   %%   * <<"min-masters">>
+   %%   * <<"client-local">>
+   %%   * <<"random">>
+   %% See https://www.rabbitmq.com/ha.html#queue-master-location
+   %%
+   %% {queue_master_locator, <<"client-local">>},
+
+   %% Batch size (number of messages) used during eager queue mirror synchronisation.
+   %% See https://www.rabbitmq.com/ha.html#batch-sync. When average message size is relatively large
+   %% (say, 10s of kilobytes or greater), reducing this value will decrease peak amount
+   %% of RAM used by newly joining nodes that need eager synchronisation.
+   %%
+   %% {mirroring_sync_batch_size, 4096},
+
+   %% Enables flow control between queue mirrors.
+   %% Disabling this can be dangerous and is not recommended.
+   %% When flow control is disablied, queue masters can outpace mirrors and not allow mirrors to catch up.
+   %% Mirrors will end up using increasingly more RAM, eventually triggering a memory alarm.
+   %%
+   %% {mirroring_flow_control, true},
+
+   %% Additional server properties to announce to connecting clients.
+   %%
+   %% {server_properties, []},
+
    %% How to respond to cluster partitions.
-   %% See http://www.rabbitmq.com/partitions.html for further details.
+   %% See http://www.rabbitmq.com/partitions.html
    %%
    %% {cluster_partition_handling, ignore},
 
@@ -300,8 +333,7 @@
 
    %% Make clustering happen *automatically* at startup - only applied
    %% to nodes that have just been reset or started for the first time.
-   %% See http://www.rabbitmq.com/clustering.html#auto-config for
-   %% further details.
+   %% See http://www.rabbitmq.com/clustering.html#auto-config
    %%
    %% {cluster_nodes, {['rabbit@my.host.com'], disc}},
 
@@ -326,56 +358,115 @@
    %%
    %% {collect_statistics_interval, 5000},
 
+   %% Enables vhosts tracing.
    %%
-   %% Misc/Advanced Options
-   %% =====================
-   %%
-   %% NB: Change these only if you understand what you are doing!
+   %% {trace_vhosts, []},
+
+   %% Explicitly enable/disable HiPE compilation.
    %%
+   %% {hipe_compile, false},
 
-   %% Explicitly enable/disable hipe compilation.
+   %% Number of delegate processes to use for intra-cluster communication.
+   %% On a node which is part of cluster, has more than 16 cores and plenty of network bandwidth,
+   %% it may make sense to increase this value.
    %%
-   %% {hipe_compile, true},
+   %% {delegate_count, 16},
 
-   %% Number of times to retry while waiting for Mnesia tables in a cluster to
-   %% become available.
+   %% Number of times to retry while waiting for internal database tables (Mnesia tables) to sync
+   %% from a peer. In deployments where nodes can take a long time to boot, this value
+   %% may need increasing.
    %%
    %% {mnesia_table_loading_retry_limit, 10},
 
-   %% Time to wait per retry for Mnesia tables in a cluster to become
-   %% available.
+   %% Amount of time in milliseconds which this node will wait for internal database tables (Mnesia tables) to sync
+   %% from a peer. In deployments where nodes can take a long time to boot, this value
+   %% may need increasing.
    %%
    %% {mnesia_table_loading_retry_timeout, 30000},
 
-   %% Size in bytes below which to embed messages in the queue index. See
-   %% http://www.rabbitmq.com/persistence-conf.html
+   %% Size in bytes below which to embed messages in the queue index.
+   %% See http://www.rabbitmq.com/persistence-conf.html
    %%
    %% {queue_index_embed_msgs_below, 4096},
 
-   %% Whether or not to enable background periodic GC of all
-   %% Erlang processes in "waiting" state.
+   %% Maximum number of queue index entries to keep in journal
+   %% See http://www.rabbitmq.com/persistence-conf.html.
    %%
-   %% Disabling background GC may reduce latency for client operations,
-   %% keeping it enabled may reduce median RAM usage.
+   %% {queue_index_max_journal_entries, 32768},
+
+   %% Number of credits that a queue process is given by the message store
+   %% By default, a queue process is given 4000 message store credits,
+   %% and then 800 for every 800 messages that it processes.
    %%
-   %% {background_gc_enabled, false},
+   %% {msg_store_credit_disc_bound, {4000, 800}},
+
+   %% Minimum number of messages with their queue position held in RAM required
+   %% to trigger writing their queue position to disk.
    %%
-   %% Target (desired) interval (in milliseconds) at which we run background GC.
-   %% The actual interval will vary depending on how long it takes to execute
-   %% the operation (can be higher than this interval). Values less than
-   %% 30000 milliseconds are not recommended.
+   %% This value MUST be higher than the initial msg_store_credit_disc_bound value,
+   %% otherwise paging performance may worsen.
+   %%
+   %% {msg_store_io_batch_size, 4096},
+
+   %% Number of credits that a connection, channel or queue are given.
+   %%
+   %% By default, every connection, channel or queue is given 400 credits,
+   %% and then 200 for every 200 messages that it sends to a peer process.
+   %% Increasing these values may help with throughput but also can be dangerous:
+   %% high credit flow values are no different from not having flow control at all.
+   %%
+   %% See https://www.rabbitmq.com/blog/2015/10/06/new-credit-flow-settings-on-rabbitmq-3-5-5/
+   %% and http://alvaro-videla.com/2013/09/rabbitmq-internals-credit-flow-for-erlang-processes.html.
+   %%
+   %% {credit_flow_default_credit, {400, 200}},
+
+   %% Number of milliseconds before a channel operation times out.
+   %%
+   %% {channel_operation_timeout, 15000},
+
+   %% Number of queue operations required to trigger an explicit garbage collection.
+   %% Increasing this value may reduce CPU load and increase peak RAM consumption of queues.
+   %%
+   %% {queue_explicit_gc_run_operation_threshold, 1000},
+
+   %% Number of lazy queue operations required to trigger an explicit garbage collection.
+   %% Increasing this value may reduce CPU load and increase peak RAM consumption of lazy queues.
+   %%
+   %% {lazy_queue_explicit_gc_run_operation_threshold, 1000},
+
+   %% Number of times disk monitor will retry free disk space queries before
+   %% giving up.
+   %%
+   %% {disk_monitor_failure_retries, 10},
+
+   %% Milliseconds to wait between disk monitor retries on failures.
+   %%
+   %% {disk_monitor_failure_retry_interval, 120000},
+
+   %% Whether or not to enable background GC.
+   %%
+   %% {background_gc_enabled, false},
+
+   %% Interval (in milliseconds) at which we run background GC.
    %%
    %% {background_gc_target_interval, 60000},
 
-   %% Whether or not to enable proxy protocol support.
-   %% Once enabled, clients cannot directly connect to the broker
-   %% anymore. They must connect through a load balancer that sends the
-   %% proxy protocol header to the broker at connection time.
-   %% This setting applies only to AMQP clients, other protocols
-   %% like MQTT or STOMP have their own setting to enable proxy protocol.
-   %% See the plugins documentation for more information.
+   %% Message store operations are stored in a sequence of files called segments.
+   %% This controls max size of a segment file.
+   %% Increasing this value may speed up (sequential) disk writes but will slow down segment GC process.
+   %% DO NOT CHANGE THIS for existing installations.
    %%
-   %% {proxy_protocol, false}
+   %% {msg_store_file_size_limit, 16777216},
+
+   %% Whether or not to enable file write buffering.
+   %%
+   %% {fhc_write_buffering, true},
+
+   %% Whether or not to enable file read buffering. Enabling
+   %% this may slightly speed up reads but will also increase
+   %% node's memory consumption, in particular on boot.
+   %%
+   %% {fhc_read_buffering, false}
 
   ]},
 
@@ -600,21 +691,27 @@
    %%
    %% {prefetch, 10},
 
-   %% TCP/SSL Configuration (as per the broker configuration).
+   %% TLS listeners.
+   %% See http://www.rabbitmq.com/networking.html
    %%
    %% {tcp_listeners, [1883]},
    %% {ssl_listeners, []},
 
    %% Number of Erlang processes that will accept connections for the TCP
-   %% and SSL listeners.
+   %% and TLS listeners.
+   %% See http://www.rabbitmq.com/networking.html
    %%
    %% {num_tcp_acceptors, 10},
    %% {num_ssl_acceptors, 1},
 
-   %% TCP/Socket options (as per the broker configuration).
+   %% TCP socket options.
+   %% See http://www.rabbitmq.com/networking.html
    %%
-   %% {tcp_listen_options, [{backlog,   128},
-   %%                       {nodelay,   true}]},
+   %% {tcp_listen_options, [
+   %%                         {backlog,   128},
+   %%                         {linger,        {true, 0}},
+   %%                         {exit_on_close, false}
+   %%                      ]},
 
    %% Whether or not to enable proxy protocol support.
    %% Once enabled, clients cannot directly connect to the broker
diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl
index 4d61bb4b03..ac3d60bb11 100644
--- a/src/rabbit_variable_queue.erl
+++ b/src/rabbit_variable_queue.erl
@@ -189,12 +189,12 @@
 %% (betas+gammas+delta)/(target_ram_count+betas+gammas+delta). I.e. as
 %% the target_ram_count shrinks to 0, so must betas and gammas.
 %%
-%% The conversion of betas to gammas is done in batches of at least
-%% ?IO_BATCH_SIZE. This value should not be too small, otherwise the
-%% frequent operations on the queues of q2 and q3 will not be
-%% effectively amortised (switching the direction of queue access
-%% defeats amortisation). Note that there is a natural upper bound due
-%% to credit_flow limits on the alpha to beta conversion.
+%% The conversion of betas to deltas is done if there are at least
+%% ?IO_BATCH_SIZE betas in q2 & q3. This value should not be too small,
+%% otherwise the frequent operations on the queues of q2 and q3 will not be
+%% effectively amortised (switching the direction of queue access defeats
+%% amortisation). Note that there is a natural upper bound due to credit_flow
+%% limits on the alpha to beta conversion.
 %%
 %% The conversion from alphas to betas is chunked due to the
 %% credit_flow limits of the msg_store. This further smooths the
@@ -2428,6 +2428,9 @@ reduce_memory_use(State = #vqstate {
         end,
 
     State3 =
+        %% If there are more messages with their queue position held in RAM,
+        %% a.k.a. betas, in Q2 & Q3 than IoBatchSize,
+        %% write their queue position to disk, a.k.a. push_betas_to_deltas
         case chunk_size(?QUEUE:len(Q2) + ?QUEUE:len(Q3),
                         permitted_beta_count(State1)) of
             S2 when S2 >= IoBatchSize ->