diff options
98 files changed, 8854 insertions, 1260 deletions
diff --git a/.gitignore b/.gitignore index 52a0257b62..7af1ad4260 100644 --- a/.gitignore +++ b/.gitignore @@ -7,12 +7,14 @@ /cover/ /debug/ /deps/ +/debug/ /doc/ /ebin/ /etc/ /logs/ /plugins/ /test/ct.cover.spec +/test/config_schema_SUITE_data/schema/** /xrefr rabbit.d @@ -39,3 +41,5 @@ docs/rabbitmqctl.1.man.xml # Tracing tools *-ttb *.ti + +PACKAGES/* diff --git a/.travis.yml b/.travis.yml index 22e67aea5f..e6bba5c81c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,6 @@ addons: packages: - xsltproc otp_release: - - "R16B03-1" - - "17.5" - "18.3" - "19.0" env: diff --git a/LICENSE-MPL-RabbitMQ b/LICENSE-MPL-RabbitMQ index 82c7cf5419..f1ba9a5ca1 100644 --- a/LICENSE-MPL-RabbitMQ +++ b/LICENSE-MPL-RabbitMQ @@ -447,7 +447,7 @@ EXHIBIT A -Mozilla Public License. The Original Code is RabbitMQ. The Initial Developer of the Original Code is Pivotal Software, Inc. - Copyright (c) 2007-2015 Pivotal Software, Inc. All rights reserved.'' + Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved.'' [NOTE: The text of this Exhibit A may differ slightly from the text of the notices in the Source Code files of the Original Code. You should @@ -1,7 +1,7 @@ PROJECT = rabbit VERSION ?= $(call get_app_version,src/$(PROJECT).app.src) -DEPS = ranch rabbit_common +DEPS = ranch lager rabbit_common TEST_DEPS = rabbitmq_ct_helpers amqp_client meck proper define usage_xml_to_erl @@ -19,7 +19,7 @@ * [Documentation guides](http://www.rabbitmq.com/documentation.html) * [Client libraries and tools](http://www.rabbitmq.com/devtools.html) * [Tutorials Source Code](https://github.com/rabbitmq/rabbitmq-tutorials/) - + ## Getting Help * [RabbitMQ mailing list](https://groups.google.com/forum/#!forum/rabbitmq-users) @@ -37,9 +37,10 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) and our [development process overview]( RabbitMQ server is [licensed under the MPL](LICENSE-MPL-RabbitMQ). -## Building From Source +## Building From Source and Packaging -See [building RabbitMQ server from source](http://www.rabbitmq.com/build-server.html). + * [Building RabbitMQ Server From Source](http://www.rabbitmq.com/build-server.html) + * [Building RabbitMQ Server Packages](http://www.rabbitmq.com/build-server.html) ## Copyright diff --git a/docs/advanced.config.example b/docs/advanced.config.example new file mode 100644 index 0000000000..82a1c000e1 --- /dev/null +++ b/docs/advanced.config.example @@ -0,0 +1,109 @@ +[ + + + %% ---------------------------------------------------------------------------- + %% Advanced Erlang Networking/Clustering Options. + %% + %% See http://www.rabbitmq.com/clustering.html for details + %% ---------------------------------------------------------------------------- + %% Sets the net_kernel tick time. + %% Please see http://erlang.org/doc/man/kernel_app.html and + %% http://www.rabbitmq.com/nettick.html for further details. + %% + %% {kernel, [{net_ticktime, 60}]}, + %% ---------------------------------------------------------------------------- + %% RabbitMQ Shovel Plugin + %% + %% See http://www.rabbitmq.com/shovel.html for details + %% ---------------------------------------------------------------------------- + + {rabbitmq_shovel, + [{shovels, + [%% A named shovel worker. + %% {my_first_shovel, + %% [ + + %% List the source broker(s) from which to consume. + %% + %% {sources, + %% [%% URI(s) and pre-declarations for all source broker(s). + %% {brokers, ["amqp://user:password@host.domain/my_vhost"]}, + %% {declarations, []} + %% ]}, + + %% List the destination broker(s) to publish to. + %% {destinations, + %% [%% A singular version of the 'brokers' element. + %% {broker, "amqp://"}, + %% {declarations, []} + %% ]}, + + %% Name of the queue to shovel messages from. + %% + %% {queue, <<"your-queue-name-goes-here">>}, + + %% Optional prefetch count. + %% + %% {prefetch_count, 10}, + + %% when to acknowledge messages: + %% - no_ack: never (auto) + %% - on_publish: after each message is republished + %% - on_confirm: when the destination broker confirms receipt + %% + %% {ack_mode, on_confirm}, + + %% Overwrite fields of the outbound basic.publish. + %% + %% {publish_fields, [{exchange, <<"my_exchange">>}, + %% {routing_key, <<"from_shovel">>}]}, + + %% Static list of basic.properties to set on re-publication. + %% + %% {publish_properties, [{delivery_mode, 2}]}, + + %% The number of seconds to wait before attempting to + %% reconnect in the event of a connection failure. + %% + %% {reconnect_delay, 2.5} + + %% ]} %% End of my_first_shovel + ]} + %% Rather than specifying some values per-shovel, you can specify + %% them for all shovels here. + %% + %% {defaults, [{prefetch_count, 0}, + %% {ack_mode, on_confirm}, + %% {publish_fields, []}, + %% {publish_properties, [{delivery_mode, 2}]}, + %% {reconnect_delay, 2.5}]} + ]}, + + {rabbitmq_auth_backend_ldap, [ + %% + %% Authorisation + %% ============= + %% + + %% The LDAP plugin can perform a variety of queries against your + %% LDAP server to determine questions of authorisation. See + %% http://www.rabbitmq.com/ldap.html#authorisation for more + %% information. + + %% Set the query to use when determining vhost access + %% + %% {vhost_access_query, {in_group, + %% "ou=${vhost}-users,ou=vhosts,dc=example,dc=com"}}, + + %% Set the query to use when determining resource (e.g., queue) access + %% + %% {resource_access_query, {constant, true}}, + + %% Set queries to determine which tags a user has + %% + %% {tag_queries, []} + ]} +]. + + + diff --git a/docs/rabbitmq.conf.example b/docs/rabbitmq.conf.example new file mode 100644 index 0000000000..f03145b447 --- /dev/null +++ b/docs/rabbitmq.conf.example @@ -0,0 +1,736 @@ +# ====================================== +# RabbbitMQ broker section +# ====================================== + +## Network Connectivity +## ==================== +## +## By default, RabbitMQ will listen on all interfaces, using +## the standard (reserved) AMQP port. +## +# listeners.tcp.default = 5672 + + +## To listen on a specific interface, provide an IP address with port. +## For example, to listen only on localhost for both IPv4 and IPv6: +## +# IPv4 +# listeners.tcp.local = 127.0.0.1:5672 +# IPv6 +# listeners.tcp.local_v6 = ::1:5672 + +## You can define multiple listeners using listener names +# listeners.tcp.other_port = 5673 +# listeners.tcp.other_ip = 10.10.10.10:5672 + + +## SSL listeners are configured in the same fashion as TCP listeners, +## including the option to control the choice of interface. +## +# listeners.ssl.default = 5671 + +## Number of Erlang processes that will accept connections for the TCP +## and SSL listeners. +## +# num_acceptors.tcp = 10 +# num_acceptors.ssl = 1 + + +## Maximum time for AMQP 0-8/0-9/0-9-1 handshake (after socket connection +## and SSL handshake), in milliseconds. +## +# handshake_timeout = 10000 + +## Set to 'true' to perform reverse DNS lookups when accepting a +## connection. Hostnames will then be shown instead of IP addresses +## in rabbitmqctl and the management plugin. +## +# reverse_dns_lookups = true + +## +## Security / AAA +## ============== +## + +## The default "guest" user is only permitted to access the server +## via a loopback interface (e.g. localhost). +## {loopback_users, [<<"guest">>]}, +## +# loopback_users.guest = true + +## Uncomment the following line if you want to allow access to the +## guest user from anywhere on the network. +# loopback_users.guest = false + +## Configuring SSL. +## See http://www.rabbitmq.com/ssl.html for full documentation. +## +# ssl_options.verify = verify_peer +# ssl_options.fail_if_no_peer_cert = false +# ssl_options.cacertfile = /path/to/cacert.pem +# ssl_options.certfile = /path/to/cert.pem +# ssl_options.keyfile = /path/to/key.pem + +## Select an authentication/authorisation backend to use. +## +## Alternative backends are provided by plugins, such as rabbitmq-auth-backend-ldap. +## +## NB: These settings require certain plugins to be enabled. +## See http://www.rabbitmq.com/plugins.html and http://rabbitmq.com/access-control.html +## for details. + +# auth_backends.1 = rabbit_auth_backend_internal + +## uses separate backends for authentication and authorisation, +## see below. +# auth_backends.1.authn = rabbit_auth_backend_ldap +# auth_backends.1.authz = rabbit_auth_backend_internal + +## The rabbitmq_auth_backend_ldap plugin allows the broker to +## perform authentication and authorisation by deferring to an +## external LDAP server. +## +## For more information about configuring the LDAP backend, see +## http://www.rabbitmq.com/ldap.html and http://rabbitmq.com/access-control.html. +## +## uses LDAP for both authentication and authorisation +# auth_backends.1 = rabbit_auth_backend_ldap + +## uses HTTP service for both authentication and +## authorisation +# auth_backends.1 = rabbit_auth_backend_http + +## uses two backends in a chain: HTTP first, then internal +# auth_backends.1 = rabbit_auth_backend_http +# auth_backends.2 = rabbit_auth_backend_internal + +## Authentication +## The built-in mechanisms are 'PLAIN', +## 'AMQPLAIN', and 'EXTERNAL' Additional mechanisms can be added via +## plugins. +## +## See http://www.rabbitmq.com/authentication.html for more details. +## +# auth_mechanisms.1 = PLAIN +# auth_mechanisms.2 = AMQPLAIN + +## The rabbitmq-auth-mechanism-ssl plugin makes it possible to +## authenticate a user based on the client's x509 (TLS) certificate. +## See http://www.rabbitmq.com/authentication.html for more info. +## +## To use auth-mechanism-ssl, the EXTERNAL mechanism should +## be enabled: +## +# auth_mechanisms.1 = PLAIN +# auth_mechanisms.2 = AMQPLAIN +# auth_mechanisms.3 = EXTERNAL + +## To force x509 certificate-based authentication on all clients, +## exclude all other mechanisms (note: this will disable password-based +## authentication even for the management UI!): +## +# auth_mechanisms.1 = EXTERNAL + +## This pertains to both the rabbitmq-auth-mechanism-ssl plugin and +## STOMP ssl_cert_login configurations. See the rabbitmq_stomp +## configuration section later in this file and the README in +## https://github.com/rabbitmq/rabbitmq-auth-mechanism-ssl for further +## details. +## +## To use the SSL cert's CN instead of its DN as the username +## +# ssl_cert_login_from = common_name + +## SSL handshake timeout, in milliseconds. +## +# ssl_handshake_timeout = 5000 + + +## Password hashing implementation. Will only affect newly +## created users. To recalculate hash for an existing user +## it's necessary to update her password. +## +## To use SHA-512, set to rabbit_password_hashing_sha512. +## +# password_hashing_module = rabbit_password_hashing_sha256 + +## When importing definitions exported from versions earlier +## than 3.6.0, it is possible to go back to MD5 (only do this +## as a temporary measure!) by setting this to rabbit_password_hashing_md5. +## +# password_hashing_module = rabbit_password_hashing_md5 + +## +## Default User / VHost +## ==================== +## + +## On first start RabbitMQ will create a vhost and a user. These +## config items control what gets created. See +## http://www.rabbitmq.com/access-control.html for further +## information about vhosts and access control. +## +# default_vhost = / +# default_user = guest +# default_pass = guest + +# default_permissions.configure = .* +# default_permissions.read = .* +# default_permissions.write = .* + +## Tags for default user +## +## For more details about tags, see the documentation for the +## Management Plugin at http://www.rabbitmq.com/management.html. +## +# default_user_tags.administrator = true + +## Define other tags like this: +# default_user_tags.management = true +# default_user_tags.custom_tag = true + +## +## Additional network and protocol related configuration +## ===================================================== +## + +## Set the default AMQP 0-9-1 heartbeat interval (in seconds). +## See http://rabbitmq.com/heartbeats.html for more details. +## +# heartbeat = 600 + +## Set the max permissible size of an AMQP frame (in bytes). +## +# frame_max = 131072 + +## Set the max frame size the server will accept before connection +## tuning occurs +## +# initial_frame_max = 4096 + +## Set the max permissible number of channels per connection. +## 0 means "no limit". +## +# channel_max = 128 + +## Customising Socket Options. +## +## See (http://www.erlang.org/doc/man/inet.html#setopts-2) for +## further documentation. +## + +# tcp_listen_options.backlog = 128 +# tcp_listen_options.nodelay = true +# tcp_listen_options.exit_on_close = false + +## +## Resource Limits & Flow Control +## ============================== +## +## See http://www.rabbitmq.com/memory.html for full details. + +## Memory-based Flow Control threshold. +## +# vm_memory_high_watermark.relative = 0.4 + +## Alternatively, we can set a limit (in bytes) of RAM used by the node. +## +# vm_memory_high_watermark.absolute = 1073741824 + +## Or you can set absolute value using memory units (with RabbitMQ 3.6.0+). +## Absolute watermark will be ignored if relative is defined! +## +# vm_memory_high_watermark.absolute = 2GB +## +## Supported units suffixes: +## +## kb, KB: kibibytes (2^10 bytes) +## mb, MB: mebibytes (2^20) +## gb, GB: gibibytes (2^30) + + + +## Fraction of the high watermark limit at which queues start to +## page message out to disc in order to free up memory. +## +## Values greater than 0.9 can be dangerous and should be used carefully. +## +# vm_memory_high_watermark_paging_ratio = 0.5 + +## Interval (in milliseconds) at which we perform the check of the memory +## levels against the watermarks. +## +# memory_monitor_interval = 2500 + +## Set disk free limit (in bytes). Once free disk space reaches this +## lower bound, a disk alarm will be set - see the documentation +## listed above for more details. +## +## Absolute watermark will be ignored if relative is defined! +# disk_free_limit.absolute = 50000 + +## Or you can set it using memory units (same as in vm_memory_high_watermark) +## with RabbitMQ 3.6.0+. +# disk_free_limit.absolute = 500KB +# disk_free_limit.absolute = 50mb +# disk_free_limit.absolute = 5GB + +## Alternatively, we can set a limit relative to total available RAM. +## +## Values lower than 1.0 can be dangerous and should be used carefully. +# disk_free_limit.relative = 2.0 + +## +## Clustering +## ===================== +## +# cluster_partition_handling = ignore + +## pause_if_all_down strategy require additional configuration +# cluster_partition_handling = pause_if_all_down + +## Recover strategy. Can be either 'autoheal' or 'ignore' +# cluster_partition_handling.pause_if_all_down.recover = ignore + +## Node names to check +# cluster_partition_handling.pause_if_all_down.nodes.1 = rabbit@localhost +# cluster_partition_handling.pause_if_all_down.nodes.2 = hare@localhost + +## Mirror sync batch size, in messages. Increasing this will speed +## up syncing but total batch size in bytes must not exceed 2 GiB. +## Available in RabbitMQ 3.6.0 or later. +## +# mirroring_sync_batch_size = 4096 + +## Make clustering happen *automatically* at startup - only applied +## to nodes that have just been reset or started for the first time. +## See http://www.rabbitmq.com/clustering.html#auto-config for +## further details. +## +# autocluster.classic_config.nodes.node1 = rabbit1@hostname +# autocluster.classic_config.nodes.node2 = rabbit2@hostname +# autocluster.classic_config.nodes.node3 = rabbit3@hostname +# autocluster.classic_config.nodes.node4 = rabbit4@hostname + +## This node's type can be configured. If you are not sure +## what node type to use, always use 'disc'. +# autocluster.classic_config.node_type = disc + +## Interval (in milliseconds) at which we send keepalive messages +## to other cluster members. Note that this is not the same thing +## as net_ticktime; missed keepalive messages will not cause nodes +## to be considered down. +## +# cluster_keepalive_interval = 10000 + +## +## Statistics Collection +## ===================== +## + +## Set (internal) statistics collection granularity. +## +## Can be none, coarse or fine +# collect_statistics = none + +# collect_statistics = coarse + +## Statistics collection interval (in milliseconds). Increasing +## this will reduce the load on management database. +## +# collect_statistics_interval = 5000 + +## +## Misc/Advanced Options +## ===================== +## +## NB: Change these only if you understand what you are doing! +## + +## Explicitly enable/disable hipe compilation. +## +# hipe_compile = false + +## Timeout used when waiting for Mnesia tables in a cluster to +## become available. +## +# mnesia_table_loading_timeout = 30000 + +## Size in bytes below which to embed messages in the queue index. See +## http://www.rabbitmq.com/persistence-conf.html +## +# queue_index_embed_msgs_below = 4096 + +## You can also set this size in memory units +## +# queue_index_embed_msgs_below = 4kb + +## ---------------------------------------------------------------------------- +## Advanced Erlang Networking/Clustering Options. +## +## See http://www.rabbitmq.com/clustering.html for details +## ---------------------------------------------------------------------------- + +# ====================================== +# Kernel section +# ====================================== + +# kernel.net_ticktime = 60 + +## ---------------------------------------------------------------------------- +## RabbitMQ Management Plugin +## +## See http://www.rabbitmq.com/management.html for details +## ---------------------------------------------------------------------------- + +# ======================================= +# Management section +# ======================================= + +## Pre-Load schema definitions from the following JSON file. See +## http://www.rabbitmq.com/management.html#load-definitions +## +# management.load_definitions = /path/to/schema.json + +## Log all requests to the management HTTP API to a file. +## +# management.http_log_dir = /path/to/access.log + +## Change the port on which the HTTP listener listens, +## specifying an interface for the web server to bind to. +## Also set the listener to use SSL and provide SSL options. +## + +# QA: Maybe use IP type like in tcp_listener? +# management.listener.port = 12345 +# management.listener.ip = 127.0.0.1 +# management.listener.ssl = true + +# management.listener.ssl_opts.cacertfile = /path/to/cacert.pem +# management.listener.ssl_opts.certfile = /path/to/cert.pem +# management.listener.ssl_opts.keyfile = /path/to/key.pem + +## One of 'basic', 'detailed' or 'none'. See +## http://www.rabbitmq.com/management.html#fine-stats for more details. +# management.rates_mode = basic + +## Configure how long aggregated data (such as message rates and queue +## lengths) is retained. Please read the plugin's documentation in +## http://www.rabbitmq.com/management.html#configuration for more +## details. +## Your can use 'minute', 'hour' and '24hours' keys or integer key (in seconds) +# management.sample_retention_policies.global.minute = 5 +# management.sample_retention_policies.global.hour = 60 +# management.sample_retention_policies.global.24hours = 1200 + +# management.sample_retention_policies.basic.minute = 5 +# management.sample_retention_policies.basic.hour = 60 + +# management.sample_retention_policies.detailed.10 = 5 + +## ---------------------------------------------------------------------------- +## RabbitMQ Shovel Plugin +## +## See http://www.rabbitmq.com/shovel.html for details +## ---------------------------------------------------------------------------- + +## Shovel plugin config example is defined in additional.config file + + +## ---------------------------------------------------------------------------- +## RabbitMQ Stomp Adapter +## +## See http://www.rabbitmq.com/stomp.html for details +## ---------------------------------------------------------------------------- + +# ======================================= +# STOMP section +# ======================================= + +## Network Configuration - the format is generally the same as for the broker +## +# stomp.listeners.tcp.default = 61613 + +## Same for ssl listeners +## +# stomp.listeners.ssl.default = 61614 + +## Number of Erlang processes that will accept connections for the TCP +## and SSL listeners. +## +# stomp.num_acceptors.tcp = 10 +# stomp.num_acceptors.ssl = 1 + +## Additional SSL options + +## Extract a name from the client's certificate when using SSL. +## +# stomp.ssl_cert_login = true + +## Set a default user name and password. This is used as the default login +## whenever a CONNECT frame omits the login and passcode headers. +## +## Please note that setting this will allow clients to connect without +## authenticating! +## +# stomp.default_user = guest +# stomp.default_pass = guest + +## If a default user is configured, or you have configured use SSL client +## certificate based authentication, you can choose to allow clients to +## omit the CONNECT frame entirely. If set to true, the client is +## automatically connected as the default user or user supplied in the +## SSL certificate whenever the first frame sent on a session is not a +## CONNECT frame. +## +# stomp.implicit_connect = true + +## ---------------------------------------------------------------------------- +## RabbitMQ MQTT Adapter +## +## See https://github.com/rabbitmq/rabbitmq-mqtt/blob/stable/README.md +## for details +## ---------------------------------------------------------------------------- + +# ======================================= +# MQTT section +# ======================================= + +## Set the default user name and password. Will be used as the default login +## if a connecting client provides no other login details. +## +## Please note that setting this will allow clients to connect without +## authenticating! +## +# mqtt.default_user = guest +# mqtt.default_pass = guest + +## Enable anonymous access. If this is set to false, clients MUST provide +## login information in order to connect. See the default_user/default_pass +## configuration elements for managing logins without authentication. +## +# mqtt.allow_anonymous = true + +## If you have multiple chosts, specify the one to which the +## adapter connects. +## +# mqtt.vhost = / + +## Specify the exchange to which messages from MQTT clients are published. +## +# mqtt.exchange = amq.topic + +## Specify TTL (time to live) to control the lifetime of non-clean sessions. +## +# mqtt.subscription_ttl = 1800000 + +## Set the prefetch count (governing the maximum number of unacknowledged +## messages that will be delivered). +## +# mqtt.prefetch = 10 + +## TCP/SSL Configuration (as per the broker configuration). +## +# mqtt.listeners.tcp.default = 1883 + +## Same for ssl listener +## +# mqtt.listeners.ssl.default = 1884 + +## Number of Erlang processes that will accept connections for the TCP +## and SSL listeners. +## +# mqtt.num_acceptors.tcp = 10 +# mqtt.num_acceptors.ssl = 1 + +## TCP/Socket options (as per the broker configuration). +## +# mqtt.tcp_listen_options.backlog = 128 +# mqtt.tcp_listen_options.nodelay = true + +## ---------------------------------------------------------------------------- +## RabbitMQ AMQP 1.0 Support +## +## See https://github.com/rabbitmq/rabbitmq-amqp1.0/blob/stable/README.md +## for details +## ---------------------------------------------------------------------------- + +# ======================================= +# AMQP_1 section +# ======================================= + + +## Connections that are not authenticated with SASL will connect as this +## account. See the README for more information. +## +## Please note that setting this will allow clients to connect without +## authenticating! +## +# amqp1_0.default_user = guest + +## Enable protocol strict mode. See the README for more information. +## +# amqp1_0.protocol_strict_mode = false + +## Lager controls logging. +## See https://github.com/basho/lager for more documentation +## +## Log direcrory, taken from the RABBITMQ_LOG_BASE env variable by default. +## +# log.dir = /var/log/rabbitmq + +## Logging to console (can be true or false) +## +# log.console = false + +## Loglevel to log to console +## +# log.console.level = info + +## Logging to file. Can be false or filename. +## Default: +# log.file = rabbit.log + +## To turn off: +# log.file = false + +## Loglevel to log to file +## +# log.file.level = info + +## File rotation config. No rotation by defualt. +## DO NOT SET rotation date to ''. Leave unset if require "" value +# log.file.rotation.date = $D0 +# log.file.rotation.size = 0 + + +## QA: Config for syslog logging +# log.syslog = false +# log.syslog.identity = rabbitmq +# log.syslog.level = info +# log.syslog.facility = daemon + + +## ---------------------------------------------------------------------------- +## RabbitMQ LDAP Plugin +## +## See http://www.rabbitmq.com/ldap.html for details. +## +## ---------------------------------------------------------------------------- + +# ======================================= +# LDAP section +# ======================================= + +## +## Connecting to the LDAP server(s) +## ================================ +## + +## Specify servers to bind to. You *must* set this in order for the plugin +## to work properly. +## +# ldap.servers.1 = your-server-name-goes-here + +## You can define multiple servers +# ldap.servers.2 = your-other-server + +## Connect to the LDAP server using SSL +## +# ldap.use_ssl = false + +## Specify the LDAP port to connect to +## +# ldap.port = 389 + +## LDAP connection timeout, in milliseconds or 'infinity' +## +# ldap.timeout = infinity + +## Or number +# ldap.timeout = 500 + +## Enable logging of LDAP queries. +## One of +## - false (no logging is performed) +## - true (verbose logging of the logic used by the plugin) +## - network (as true, but additionally logs LDAP network traffic) +## +## Defaults to false. +## +# ldap.log = false + +## Also can be true or network +# ldap.log = true +# ldap.log = network + +## +## Authentication +## ============== +## + +## Pattern to convert the username given through AMQP to a DN before +## binding +## +# ldap.user_dn_pattern = cn=${username},ou=People,dc=example,dc=com + +## Alternatively, you can convert a username to a Distinguished +## Name via an LDAP lookup after binding. See the documentation for +## full details. + +## When converting a username to a dn via a lookup, set these to +## the name of the attribute that represents the user name, and the +## base DN for the lookup query. +## +# ldap.dn_lookup_attribute = userPrincipalName +# ldap.dn_lookup_base = DC=gopivotal,DC=com + +## Controls how to bind for authorisation queries and also to +## retrieve the details of users logging in without presenting a +## password (e.g., SASL EXTERNAL). +## One of +## - as_user (to bind as the authenticated user - requires a password) +## - anon (to bind anonymously) +## - {UserDN, Password} (to bind with a specified user name and password) +## +## Defaults to 'as_user'. +## +# ldap.other_bind = as_user + +## Or can be more complex: +# ldap.other_bind.user_dn = User +# ldap.other_bind.password = Password + +## If user_dn and password defined - other options is ignored. + +# ----------------------------- +# Too complex section of LDAP +# ----------------------------- + +## +## Authorisation +## ============= +## + +## The LDAP plugin can perform a variety of queries against your +## LDAP server to determine questions of authorisation. See +## http://www.rabbitmq.com/ldap.html#authorisation for more +## information. + +## Following configuration should be defined in additional.config file +## DO NOT UNCOMMENT THIS LINES! + +## Set the query to use when determining vhost access +## +## {vhost_access_query, {in_group, +## "ou=${vhost}-users,ou=vhosts,dc=example,dc=com"}}, + +## Set the query to use when determining resource (e.g., queue) access +## +## {resource_access_query, {constant, true}}, + +## Set queries to determine which tags a user has +## +## {tag_queries, []} +# ]}, +# ----------------------------- diff --git a/docs/rabbitmq.config.example b/docs/rabbitmq.config.example index f425726721..b31ec4d673 100644 --- a/docs/rabbitmq.config.example +++ b/docs/rabbitmq.config.example @@ -38,12 +38,6 @@ %% %% {handshake_timeout, 10000}, - %% Log levels (currently just used for connection logging). - %% One of 'debug', 'info', 'warning', 'error' or 'none', in decreasing - %% order of verbosity. Defaults to 'info'. - %% - %% {log_levels, [{connection, info}, {channel, info}]}, - %% Set to 'true' to perform reverse DNS lookups when accepting a %% connection. Hostnames will then be shown instead of IP addresses %% in rabbitmqctl and the management plugin. @@ -130,6 +124,12 @@ %% created users. To recalculate hash for an existing user %% it's necessary to update her password. %% + %% When importing definitions exported from versions earlier + %% than 3.6.0, it is possible to go back to MD5 (only do this + %% as a temporary measure!) by setting this to rabbit_password_hashing_md5. + %% + %% To use SHA-512, set to rabbit_password_hashing_sha512. + %% %% {password_hashing_module, rabbit_password_hashing_sha256}, %% Configuration entry encryption. @@ -221,7 +221,7 @@ %% %% {vm_memory_high_watermark, {absolute, 1073741824}}, %% - %% Or you can set absolute value using memory units. + %% Or you can set absolute value using memory units (with RabbitMQ 3.6.0+). %% %% {vm_memory_high_watermark, {absolute, "1024M"}}, %% @@ -253,6 +253,7 @@ %% {disk_free_limit, 50000000}, %% %% Or you can set it using memory units (same as in vm_memory_high_watermark) + %% with RabbitMQ 3.6.0+. %% {disk_free_limit, "50MB"}, %% {disk_free_limit, "50000kB"}, %% {disk_free_limit, "2GB"}, @@ -263,20 +264,20 @@ %% {disk_free_limit, {mem_relative, 2.0}}, %% - %% Misc/Advanced Options + %% Clustering %% ===================== %% - %% NB: Change these only if you understand what you are doing! - %% - - %% To announce custom properties to clients on connection: - %% - %% {server_properties, []}, %% How to respond to cluster partitions. %% See http://www.rabbitmq.com/partitions.html for further details. %% %% {cluster_partition_handling, ignore}, + + %% Mirror sync batch size, in messages. Increasing this will speed + %% up syncing but total batch size in bytes must not exceed 2 GiB. + %% Available in RabbitMQ 3.6.0 or later. + %% + %% {mirroring_sync_batch_size, 4096}, %% Make clustering happen *automatically* at startup - only applied %% to nodes that have just been reset or started for the first time. @@ -292,14 +293,27 @@ %% %% {cluster_keepalive_interval, 10000}, + %% + %% Statistics Collection + %% ===================== + %% + %% Set (internal) statistics collection granularity. %% %% {collect_statistics, none}, - %% Statistics collection interval (in milliseconds). + %% Statistics collection interval (in milliseconds). Increasing + %% this will reduce the load on management database. %% %% {collect_statistics_interval, 5000}, + %% + %% Misc/Advanced Options + %% ===================== + %% + %% NB: Change these only if you understand what you are doing! + %% + %% Explicitly enable/disable hipe compilation. %% %% {hipe_compile, true}, @@ -660,5 +674,45 @@ %% Set queries to determine which tags a user has %% %% {tag_queries, []} + ]}, + + %% Lager controls logging. + %% See https://github.com/basho/lager for more documentation + {lager, [ + %% + %% Log directory, taken from the RABBITMQ_LOG_BASE env variable by default. + %% {log_root, "/var/log/rabbitmq"}, + %% + %% All log messages go to the default "sink" configured with + %% the `handlers` parameter. By default, it has a single + %% lager_file_backend handler writing messages to "$nodename.log" + %% (ie. the value of $RABBIT_LOGS). + %% {handlers, [ + %% {lager_file_backend, [{file, "rabbit.log"}, + %% {level, info}, + %% {date, ""}, + %% {size, 0}]} + %% ]}, + %% + %% Extra sinks are used in RabbitMQ to categorize messages. By + %% default, those extra sinks are configured to forward messages + %% to the default sink (see above). "rabbit_log_lager_event" + %% is the default category where all RabbitMQ messages without + %% a category go. Messages in the "channel" category go to the + %% "rabbit_channel_lager_event" Lager extra sink, and so on. + %% {extra_sinks, [ + %% {rabbit_log_lager_event, [{handlers, [ + %% {lager_forwarder_backend, + %% [lager_event, info]}]}]}, + %% {rabbit_channel_lager_event, [{handlers, [ + %% {lager_forwarder_backend, + %% [lager_event, info]}]}]}, + %% {rabbit_conection_lager_event, [{handlers, [ + %% {lager_forwarder_backend, + %% [lager_event, info]}]}]}, + %% {rabbit_mirroring_lager_event, [{handlers, [ + %% {lager_forwarder_backend, + %% [lager_event, info]}]}]} + %% ]} ]} ]. diff --git a/docs/rabbitmqctl.1.xml b/docs/rabbitmqctl.1.xml index 217d2d93ca..fc4927ee1c 100644 --- a/docs/rabbitmqctl.1.xml +++ b/docs/rabbitmqctl.1.xml @@ -261,32 +261,27 @@ </varlistentry> <varlistentry> - <term><cmdsynopsis><command>rotate_logs</command> <arg choice="req"><replaceable>suffix</replaceable></arg></cmdsynopsis></term> + <term><cmdsynopsis><command>rotate_logs</command></cmdsynopsis></term> <listitem> <para> - Instruct the RabbitMQ node to rotate the log files. + Instruct the RabbitMQ node to perform internal log rotation. </para> <para> - The RabbitMQ broker appends the contents of its log - files to files with names composed of the original name - and the suffix, and then resumes logging to freshly - created files at the original location. I.e. effectively - the current log contents are moved to the end of the - suffixed files. + Log rotation is performed according to lager settings + specified in configuration file. </para> <para> - When the target files do not exist they are created. When - no <option>suffix</option> is specified, no rotation takes - place - log files are just re-opened. + Note that there is no need to call this command in case of + external log rotation (e.g. from logrotate), because lager + detects renames and automatically reopens log files. </para> <para role="example-prefix">For example:</para> - <screen role="example">rabbitmqctl rotate_logs .1</screen> + <screen role="example">rabbitmqctl rotate_logs</screen> <para role="example"> - This command instructs the RabbitMQ node to append the contents - of the log files to files with names consisting of the original logs' - names and ".1" suffix, e.g. rabbit@mymachine.log.1 and - rabbit@mymachine-sasl.log.1. Finally, logging resumes to - fresh files at the old locations. + This command starts internal log rotation + process. Rotation is performed asynchronously, so there is + no guarantee that it will be completed when this command + returns. </para> </listitem> </varlistentry> @@ -1214,6 +1209,80 @@ </para> </listitem> </varlistentry> + <varlistentry> + <term><cmdsynopsis><command>set_operator_policy</command> <arg choice="opt">-p <replaceable>vhost</replaceable></arg> <arg choice="opt">--priority <replaceable>priority</replaceable></arg> <arg choice="opt">--apply-to <replaceable>apply-to</replaceable></arg> <arg choice="req"><replaceable>name</replaceable></arg> <arg choice="req"><replaceable>pattern</replaceable></arg> <arg choice="req"><replaceable>definition</replaceable></arg></cmdsynopsis></term> + <listitem> + <para> + Sets an operator policy that overrides a subset of arguments in user policies. Arguments are identical to those of <command>set_policy</command>. + Supported arguments: <command>expires</command>, <command>message-ttl</command>, <command>max-length</command>, and <command>max-length-bytes</command>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term><cmdsynopsis><command>clear_operator_policy</command> <arg choice="opt">-p <replaceable>vhost</replaceable></arg> <arg choice="req"><replaceable>name</replaceable></arg></cmdsynopsis></term> + <listitem> + <para> + Clears an operator policy. Arguments are identical to those of <command>clear_policy</command>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term><cmdsynopsis><command>list_operator_policies</command> <arg choice="opt">-p <replaceable>vhost</replaceable></arg></cmdsynopsis></term> + <listitem> + <para> + Lists operator policy overrides for a virtual host. Arguments are identical to those of <command>list_policies</command>. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect2> + + <refsect2> + <title>Virtual Host Limits</title> + <para> + It is possible to enforce certain limits on virtual hosts. + </para> + <variablelist> + <varlistentry> + <term><cmdsynopsis><command>set_vhost_limits</command> <arg choice="opt">-p <replaceable>vhostpath</replaceable></arg> <arg choice="req"><replaceable>definition</replaceable></arg></cmdsynopsis></term> + <listitem> + <para> + Sets virtual host limits + </para> + <variablelist> + <varlistentry> + <term>definition</term> + <listitem><para> + The definition of the limits, as a + JSON term. In most shells you are very likely to + need to quote this. + + Recognised limits: max-connections (0 means "no limit"). + </para></listitem> + </varlistentry> + </variablelist> + <para role="example-prefix">For example:</para> + <screen role="example">rabbitmqctl set_vhost_limits -p qa_env '{"max-connections": 1024}'</screen> + <para role="example"> + This command limits the max number of concurrent connections in vhost <command>qa_env</command> + to 1024. + </para> + </listitem> + </varlistentry> + +<varlistentry> + <term><cmdsynopsis><command>clear_vhost_limits</command> <arg choice="opt">-p <replaceable>vhostpath</replaceable></arg></cmdsynopsis></term> + <listitem> + <para> + Clears virtual host limits + </para> + <para role="example-prefix">For example:</para> + <screen role="example">rabbitmqctl clear_vhost_limits -p qa_env</screen> + <para role="example"> + This command clears vhost limits in vhost <command>qa_env</command>. + </para> + </listitem> + </varlistentry> </variablelist> </refsect2> @@ -2074,9 +2143,9 @@ <varlistentry> <term>fraction</term> <listitem><para> - Limit relative to the total amount available RAM - as a non-negative floating point number. - Values lower than 1.0 can be dangerous and + Limit relative to the total amount available RAM + as a non-negative floating point number. + Values lower than 1.0 can be dangerous and should be used carefully. </para></listitem> </varlistentry> diff --git a/include/rabbit_log.hrl b/include/rabbit_log.hrl new file mode 100644 index 0000000000..fcb72e2e31 --- /dev/null +++ b/include/rabbit_log.hrl @@ -0,0 +1 @@ +-define(LAGER_SINK, rabbit_log_lager_event). diff --git a/priv/schema/.gitignore b/priv/schema/.gitignore new file mode 100644 index 0000000000..68e5b59a44 --- /dev/null +++ b/priv/schema/.gitignore @@ -0,0 +1,4 @@ +# plugin schemas are extracted +# into this directory: this is a Cuttlefish +# requirement. So we ignore them. +rabbitmq_*.schema diff --git a/priv/schema/rabbitmq.schema b/priv/schema/rabbitmq.schema new file mode 100644 index 0000000000..687101dd74 --- /dev/null +++ b/priv/schema/rabbitmq.schema @@ -0,0 +1,987 @@ +% ============================== +% Rabbit app section +% ============================== + +%% +%% Network Connectivity +%% ==================== +%% + +%% By default, RabbitMQ will listen on all interfaces, using +%% the standard (reserved) AMQP port. +%% +%% {tcp_listeners, [5672]}, +%% To listen on a specific interface, provide a tuple of {IpAddress, Port}. +%% For example, to listen only on localhost for both IPv4 and IPv6: +%% +%% {tcp_listeners, [{"127.0.0.1", 5672}, +%% {"[::1]", 5672}]}, + +{mapping, "listeners.tcp", "rabbit.tcp_listeners",[ + {datatype, {enum, [none]}} +]}. + +{mapping, "listeners.tcp.$name", "rabbit.tcp_listeners",[ + {datatype, [integer, ip]} +]}. + +{translation, "rabbit.tcp_listeners", +fun(Conf) -> + case cuttlefish:conf_get("listeners.tcp", Conf, undefined) of + none -> []; + _ -> + Settings = cuttlefish_variable:filter_by_prefix("listeners.tcp", Conf), + [ V || {_, V} <- Settings ] + end +end}. + +%% SSL listeners are configured in the same fashion as TCP listeners, +%% including the option to control the choice of interface. +%% +%% {ssl_listeners, [5671]}, + +{mapping, "listeners.ssl", "rabbit.ssl_listeners",[ + {datatype, {enum, [none]}} +]}. + +{mapping, "listeners.ssl.$name", "rabbit.ssl_listeners",[ + {datatype, [integer, ip]} +]}. + +{translation, "rabbit.ssl_listeners", +fun(Conf) -> + case cuttlefish:conf_get("listeners.ssl", Conf, undefined) of + none -> []; + _ -> + Settings = cuttlefish_variable:filter_by_prefix("listeners.ssl", Conf), + [ V || {_, V} <- Settings ] + end +end}. + +%% Number of Erlang processes that will accept connections for the TCP +%% and SSL listeners. +%% +%% {num_tcp_acceptors, 10}, +%% {num_ssl_acceptors, 1}, + +{mapping, "num_acceptors.ssl", "rabbit.num_ssl_acceptors", [ + {datatype, integer} +]}. + +{mapping, "num_acceptors.tcp", "rabbit.num_tcp_acceptors", [ + {datatype, integer} +]}. + + +%% Maximum time for AMQP 0-8/0-9/0-9-1 handshake (after socket connection +%% and SSL handshake), in milliseconds. +%% +%% {handshake_timeout, 10000}, + +{mapping, "handshake_timeout", "rabbit.handshake_timeout", [ + {datatype, integer} +]}. + +%% Set to 'true' to perform reverse DNS lookups when accepting a +%% connection. Hostnames will then be shown instead of IP addresses +%% in rabbitmqctl and the management plugin. +%% +%% {reverse_dns_lookups, true}, + +{mapping, "reverse_dns_lookups", "rabbit.reverse_dns_lookups", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "erlang.K", "vm_args.+K", [ + {default, "true"}, + {level, advanced} +]}. + +%% +%% Security / AAA +%% ============== +%% + +%% The default "guest" user is only permitted to access the server +%% via a loopback interface (e.g. localhost). +%% {loopback_users, [<<"guest">>]}, +%% +%% Uncomment the following line if you want to allow access to the +%% guest user from anywhere on the network. +%% {loopback_users, []}, + +{mapping, "loopback_users", "rabbit.loopback_users", [ + {datatype, {enum, [none]}} +]}. + +{mapping, "loopback_users.$user", "rabbit.loopback_users", [ + {datatype, atom} +]}. + +{translation, "rabbit.loopback_users", +fun(Conf) -> + None = cuttlefish:conf_get("loopback_users", Conf, undefined), + case None of + none -> []; + _ -> + Settings = cuttlefish_variable:filter_by_prefix("loopback_users", Conf), + [ list_to_binary(U) || {["loopback_users", U], V} <- Settings, V == true ] + end +end}. + +%% Configuring SSL. +%% See http://www.rabbitmq.com/ssl.html for full documentation. +%% +%% {ssl_options, [{cacertfile, "/path/to/testca/cacert.pem"}, +%% {certfile, "/path/to/server/cert.pem"}, +%% {keyfile, "/path/to/server/key.pem"}, +%% {verify, verify_peer}, +%% {fail_if_no_peer_cert, false}]}, + +%% SSL options section ======================================================== + +{mapping, "ssl_allow_poodle_attack", "rabbit.ssl_allow_poodle_attack", +[{datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options", "rabbit.ssl_options", [ + {datatype, {enum, [none]}} +]}. + +{translation, "rabbit.ssl_options", +fun(Conf) -> + case cuttlefish:conf_get("ssl_options", Conf, undefined) of + none -> []; + _ -> cuttlefish:invalid("Invalid ssl_options") + end +end}. + +{mapping, "ssl_options.verify", "rabbit.ssl_options.verify", [ + {datatype, {enum, [verify_peer, verify_none]}}]}. + +{mapping, "ssl_options.fail_if_no_peer_cert", "rabbit.ssl_options.fail_if_no_peer_cert", [ + {datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options.cacertfile", "rabbit.ssl_options.cacertfile", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "ssl_options.certfile", "rabbit.ssl_options.certfile", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "ssl_options.cacerts.$name", "rabbit.ssl_options.cacerts", + [{datatype, string}]}. + +{translation, "rabbit.ssl_options.cacerts", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("ssl_options.cacerts", Conf), + [ list_to_binary(V) || {_, V} <- Settings ] +end}. + +{mapping, "ssl_options.cert", "rabbit.ssl_options.cert", + [{datatype, string}]}. + +{translation, "rabbit.ssl_options.cert", +fun(Conf) -> + list_to_binary(cuttlefish:conf_get("ssl_options.cert", Conf)) +end}. + +{mapping, "ssl_options.client_renegotiation", "rabbit.ssl_options.client_renegotiation", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options.crl_check", "rabbit.ssl_options.crl_check", + [{datatype, [{enum, [true, false, peer, best_effort]}]}]}. + +{mapping, "ssl_options.depth", "rabbit.ssl_options.depth", + [{datatype, integer}, {validators, ["byte"]}]}. + +{mapping, "ssl_options.dh", "rabbit.ssl_options.dh", + [{datatype, string}]}. + +{translation, "rabbit.ssl_options.dh", +fun(Conf) -> + list_to_binary(cuttlefish:conf_get("ssl_options.dh", Conf)) +end}. + +{mapping, "ssl_options.dhfile", "rabbit.ssl_options.dhfile", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "ssl_options.honor_cipher_order", "rabbit.ssl_options.honor_cipher_order", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options.key.RSAPrivateKey", "rabbit.ssl_options.key", + [{datatype, string}]}. + +{mapping, "ssl_options.key.DSAPrivateKey", "rabbit.ssl_options.key", + [{datatype, string}]}. + +{mapping, "ssl_options.key.PrivateKeyInfo", "rabbit.ssl_options.key", + [{datatype, string}]}. + +{translation, "rabbit.ssl_options.key", +fun(Conf) -> + case cuttlefish_variable:filter_by_prefix("ssl_options.key", Conf) of + [{[_,_,Key], Val}|_] -> {list_to_atom(Key), list_to_binary(Val)}; + _ -> undefined + end +end}. + +{mapping, "ssl_options.keyfile", "rabbit.ssl_options.keyfile", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "ssl_options.log_alert", "rabbit.ssl_options.log_alert", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options.password", "rabbit.ssl_options.password", + [{datatype, string}]}. + +{mapping, "ssl_options.psk_identity", "rabbit.ssl_options.psk_identity", + [{datatype, string}]}. + +{mapping, "ssl_options.reuse_sessions", "rabbit.ssl_options.reuse_sessions", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options.secure_renegotiate", "rabbit.ssl_options.secure_renegotiate", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "ssl_options.versions.$version", "rabbit.ssl_options.versions", + [{datatype, atom}]}. + +{translation, "rabbit.ssl_options.versions", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("ssl_options.versions", Conf), + [ V || {_, V} <- Settings ] +end}. + +%% =========================================================================== + +%% Choose the available SASL mechanism(s) to expose. +%% The two default (built in) mechanisms are 'PLAIN' and +%% 'AMQPLAIN'. Additional mechanisms can be added via +%% plugins. +%% +%% See http://www.rabbitmq.com/authentication.html for more details. +%% +%% {auth_mechanisms, ['PLAIN', 'AMQPLAIN']}, + +{mapping, "auth_mechanisms.$name", "rabbit.auth_mechanisms", [ + {datatype, atom}]}. + +{translation, "rabbit.auth_mechanisms", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("auth_mechanisms", Conf), + [ V || {_, V} <- Settings ] +end}. + + +%% Select an authentication backend to use. RabbitMQ provides an +%% internal backend in the core. +%% +%% {auth_backends, [rabbit_auth_backend_internal]}, + +{translation, "rabbit.auth_backends", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("auth_backends", Conf), + BackendModule = fun + (internal) -> rabbit_auth_backend_internal; + (ldap) -> rabbit_auth_backend_ldap; + (http) -> rabbit_auth_backend_http; + (amqp) -> rabbit_auth_backend_amqp; + (dummy) -> rabbit_auth_backend_dummy; + (Other) when is_atom(Other) -> Other; + (_) -> cuttlefish:invalid("Unknown/unsupported auth backend") + end, + AuthBackends = [{Num, {default, BackendModule(V)}} || {["auth_backends", Num], V} <- Settings], + AuthNBackends = [{Num, {authn, BackendModule(V)}} || {["auth_backends", Num, "authn"], V} <- Settings], + AuthZBackends = [{Num, {authz, BackendModule(V)}} || {["auth_backends", Num, "authz"], V} <- Settings], + Backends = lists:foldl( + fun({NumStr, {Type, V}}, Acc) -> + Num = case catch list_to_integer(NumStr) of + N when is_integer(N) -> N; + Err -> + cuttlefish:invalid( + iolist_to_binary(io_lib:format( + "Auth backend position in the chain should be an integer ~p", [Err]))) + end, + NewVal = case dict:find(Num, Acc) of + {ok, {AuthN, AuthZ}} -> + case {Type, AuthN, AuthZ} of + {authn, undefined, _} -> + {V, AuthZ}; + {authz, _, undefined} -> + {AuthN, V}; + _ -> + cuttlefish:invalid( + iolist_to_binary( + io_lib:format( + "Auth backend already defined for the ~pth ~p backend", + [Num, Type]))) + end; + error -> + case Type of + authn -> {V, undefined}; + authz -> {undefined, V}; + default -> {V, V} + end + end, + dict:store(Num, NewVal, Acc) + end, + dict:new(), + AuthBackends ++ AuthNBackends ++ AuthZBackends), + lists:map( + fun + ({Num, {undefined, AuthZ}}) -> + cuttlefish:warn( + io_lib:format( + "Auth backend undefined for the ~pth authz backend. Using ~p", + [Num, AuthZ])), + {AuthZ, AuthZ}; + ({Num, {AuthN, undefined}}) -> + cuttlefish:warn( + io_lib:format( + "Authz backend undefined for the ~pth authn backend. Using ~p", + [Num, AuthN])), + {AuthN, AuthN}; + ({_Num, {Auth, Auth}}) -> Auth; + ({_Num, {AuthN, AuthZ}}) -> {AuthN, AuthZ} + end, + lists:keysort(1, dict:to_list(Backends))) +end}. + +{mapping, "auth_backends.$num", "rabbit.auth_backends", [ + {datatype, atom} +]}. + +{mapping, "auth_backends.$num.authn", "rabbit.auth_backends",[ + {datatype, atom} +]}. + +{mapping, "auth_backends.$num.authz", "rabbit.auth_backends",[ + {datatype, atom} +]}. + +%% This pertains to both the rabbitmq_auth_mechanism_ssl plugin and +%% STOMP ssl_cert_login configurations. See the rabbitmq_stomp +%% configuration section later in this file and the README in +%% https://github.com/rabbitmq/rabbitmq-auth-mechanism-ssl for further +%% details. +%% +%% To use the SSL cert's CN instead of its DN as the username +%% +%% {ssl_cert_login_from, common_name}, + +{mapping, "ssl_cert_login_from", "rabbit.ssl_cert_login_from", [ + {datatype, {enum, [distinguished_name, common_name]}} +]}. + +%% SSL handshake timeout, in milliseconds. +%% +%% {ssl_handshake_timeout, 5000}, + +{mapping, "ssl_handshake_timeout", "rabbit.ssl_handshake_timeout", [ + {datatype, integer} +]}. + +%% Password hashing implementation. Will only affect newly +%% created users. To recalculate hash for an existing user +%% it's necessary to update her password. +%% +%% When importing definitions exported from versions earlier +%% than 3.6.0, it is possible to go back to MD5 (only do this +%% as a temporary measure!) by setting this to rabbit_password_hashing_md5. +%% +%% To use SHA-512, set to rabbit_password_hashing_sha512. +%% +%% {password_hashing_module, rabbit_password_hashing_sha256}, + +{mapping, "password_hashing_module", "rabbit.password_hashing_module", [ + {datatype, atom} +]}. + +%% +%% Default User / VHost +%% ==================== +%% + +%% On first start RabbitMQ will create a vhost and a user. These +%% config items control what gets created. See +%% http://www.rabbitmq.com/access-control.html for further +%% information about vhosts and access control. +%% +%% {default_vhost, <<"/">>}, +%% {default_user, <<"guest">>}, +%% {default_pass, <<"guest">>}, +%% {default_permissions, [<<".*">>, <<".*">>, <<".*">>]}, + +{mapping, "default_vhost", "rabbit.default_vhost", [ + {datatype, string} +]}. + +{translation, "rabbit.default_vhost", +fun(Conf) -> + list_to_binary(cuttlefish:conf_get("default_vhost", Conf)) +end}. + +{mapping, "default_user", "rabbit.default_user", [ + {datatype, string} +]}. + +{translation, "rabbit.default_user", +fun(Conf) -> + list_to_binary(cuttlefish:conf_get("default_user", Conf)) +end}. + +{mapping, "default_pass", "rabbit.default_pass", [ + {datatype, string} +]}. + +{translation, "rabbit.default_pass", +fun(Conf) -> + list_to_binary(cuttlefish:conf_get("default_pass", Conf)) +end}. + +{mapping, "default_permissions.configure", "rabbit.default_permissions", [ + {datatype, string} +]}. + +{mapping, "default_permissions.read", "rabbit.default_permissions", [ + {datatype, string} +]}. + +{mapping, "default_permissions.write", "rabbit.default_permissions", [ + {datatype, string} +]}. + +{translation, "rabbit.default_permissions", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("default_permissions", Conf), + Configure = proplists:get_value(["default_permissions", "configure"], Settings), + Read = proplists:get_value(["default_permissions", "read"], Settings), + Write = proplists:get_value(["default_permissions", "write"], Settings), + [list_to_binary(Configure), list_to_binary(Read), list_to_binary(Write)] +end}. + +%% Tags for default user +%% +%% For more details about tags, see the documentation for the +%% Management Plugin at http://www.rabbitmq.com/management.html. +%% +%% {default_user_tags, [administrator]}, + +{mapping, "default_user_tags.$tag", "rabbit.default_user_tags", + [{datatype, {enum, [true, false]}}]}. + +{translation, "rabbit.default_user_tags", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("default_user_tags", Conf), + [ list_to_atom(Key) || {[_,Key], Val} <- Settings, Val == true ] +end}. + +%% +%% Additional network and protocol related configuration +%% ===================================================== +%% + +%% Set the default AMQP heartbeat delay (in seconds). +%% +%% {heartbeat, 600}, + +{mapping, "heartbeat", "rabbit.heartbeat", [{datatype, integer}]}. + +%% Set the max permissible size of an AMQP frame (in bytes). +%% +%% {frame_max, 131072}, + +{mapping, "frame_max", "rabbit.frame_max", [{datatype, bytesize}]}. + +%% Set the max frame size the server will accept before connection +%% tuning occurs +%% +%% {initial_frame_max, 4096}, + +{mapping, "initial_frame_max", "rabbit.initial_frame_max", [{datatype, bytesize}]}. + +%% Set the max permissible number of channels per connection. +%% 0 means "no limit". +%% +%% {channel_max, 128}, + +{mapping, "channel_max", "rabbit.channel_max", [{datatype, integer}]}. + +%% Customising Socket Options. +%% +%% See (http://www.erlang.org/doc/man/inet.html#setopts-2) for +%% further documentation. +%% +%% {tcp_listen_options, [{backlog, 128}, +%% {nodelay, true}, +%% {exit_on_close, false}]}, + +%% TCP listener section ====================================================== + +{mapping, "tcp_listen_options", "rabbit.tcp_listen_options", [ + {datatype, {enum, [none]}}]}. + +{translation, "rabbit.tcp_listen_options", +fun(Conf) -> + case cuttlefish:conf_get("tcp_listen_options", Conf, undefined) of + none -> []; + _ -> cuttlefish:invalid("Invalid tcp_listen_options") + end +end}. + +{mapping, "tcp_listen_options.backlog", "rabbit.tcp_listen_options.backlog", [ + {datatype, integer} +]}. + +{mapping, "tcp_listen_options.nodelay", "rabbit.tcp_listen_options.nodelay", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "tcp_listen_options.buffer", "rabbit.tcp_listen_options.buffer", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.delay_send", "rabbit.tcp_listen_options.delay_send", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "tcp_listen_options.dontroute", "rabbit.tcp_listen_options.dontroute", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "tcp_listen_options.exit_on_close", "rabbit.tcp_listen_options.exit_on_close", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "tcp_listen_options.fd", "rabbit.tcp_listen_options.fd", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.high_msgq_watermark", "rabbit.tcp_listen_options.high_msgq_watermark", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.high_watermark", "rabbit.tcp_listen_options.high_watermark", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.keepalive", "rabbit.tcp_listen_options.keepalive", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "tcp_listen_options.low_msgq_watermark", "rabbit.tcp_listen_options.low_msgq_watermark", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.low_watermark", "rabbit.tcp_listen_options.low_watermark", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.port", "rabbit.tcp_listen_options.port", + [{datatype, integer}, {validators, ["port"]}]}. + +{mapping, "tcp_listen_options.priority", "rabbit.tcp_listen_options.priority", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.recbuf", "rabbit.tcp_listen_options.recbuf", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.send_timeout", "rabbit.tcp_listen_options.send_timeout", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.send_timeout_close", "rabbit.tcp_listen_options.send_timeout_close", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "tcp_listen_options.sndbuf", "rabbit.tcp_listen_options.sndbuf", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.tos", "rabbit.tcp_listen_options.tos", + [{datatype, integer}]}. + +{mapping, "tcp_listen_options.linger.on", "rabbit.tcp_listen_options.linger", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "tcp_listen_options.linger.timeout", "rabbit.tcp_listen_options.linger", + [{datatype, integer}, {validators, ["non_negative_integer"]}]}. + +{translation, "rabbit.tcp_listen_options.linger", +fun(Conf) -> + LingerOn = cuttlefish:conf_get("tcp_listen_options.linger.on", Conf, false), + LingerTimeout = cuttlefish:conf_get("tcp_listen_options.linger.timeout", Conf, 0), + {LingerOn, LingerTimeout} +end}. + + +%% ========================================================================== + +%% +%% Resource Limits & Flow Control +%% ============================== +%% +%% See http://www.rabbitmq.com/memory.html for full details. + +%% Memory-based Flow Control threshold. +%% +%% {vm_memory_high_watermark, 0.4}, + +%% Alternatively, we can set a limit (in bytes) of RAM used by the node. +%% +%% {vm_memory_high_watermark, {absolute, 1073741824}}, +%% +%% Or you can set absolute value using memory units (with RabbitMQ 3.6.0+). +%% +%% {vm_memory_high_watermark, {absolute, "1024M"}}, +%% +%% Supported units suffixes: +%% +%% kb, KB: kibibytes (2^10 bytes) +%% mb, MB: mebibytes (2^20) +%% gb, GB: gibibytes (2^30) + +{mapping, "vm_memory_high_watermark.relative", "rabbit.vm_memory_high_watermark", [ + {datatype, float}]}. + +{mapping, "vm_memory_high_watermark.absolute", "rabbit.vm_memory_high_watermark", [ + {datatype, [integer, string]}]}. + + +{translation, "rabbit.vm_memory_high_watermark", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("vm_memory_high_watermark", Conf), + Absolute = proplists:get_value(["vm_memory_high_watermark", "absolute"], Settings), + Relative = proplists:get_value(["vm_memory_high_watermark", "relative"], Settings), + case {Absolute, Relative} of + {undefined, undefined} -> cuttlefish:invalid("No vm watermark defined"); + {_, undefined} -> {absolute, Absolute}; + _ -> Relative + end +end}. + +%% Fraction of the high watermark limit at which queues start to +%% page message out to disc in order to free up memory. +%% +%% Values greater than 0.9 can be dangerous and should be used carefully. +%% +%% {vm_memory_high_watermark_paging_ratio, 0.5}, + +{mapping, "vm_memory_high_watermark_paging_ratio", + "rabbit.vm_memory_high_watermark_paging_ratio", + [{datatype, float}, {validators, ["less_than_1"]}]}. + +%% Interval (in milliseconds) at which we perform the check of the memory +%% levels against the watermarks. +%% +%% {memory_monitor_interval, 2500}, + +{mapping, "memory_monitor_interval", "rabbit.memory_monitor_interval", + [{datatype, integer}]}. + +%% Set disk free limit (in bytes). Once free disk space reaches this +%% lower bound, a disk alarm will be set - see the documentation +%% listed above for more details. +%% +%% {disk_free_limit, 50000000}, +%% +%% Or you can set it using memory units (same as in vm_memory_high_watermark) +%% with RabbitMQ 3.6.0+. +%% {disk_free_limit, "50MB"}, +%% {disk_free_limit, "50000kB"}, +%% {disk_free_limit, "2GB"}, + +%% Alternatively, we can set a limit relative to total available RAM. +%% +%% Values lower than 1.0 can be dangerous and should be used carefully. +%% {disk_free_limit, {mem_relative, 2.0}}, + +{mapping, "disk_free_limit.relative", "rabbit.disk_free_limit", [ + {datatype, float}]}. + +{mapping, "disk_free_limit.absolute", "rabbit.disk_free_limit", [ + {datatype, [integer, string]}]}. + + +{translation, "rabbit.disk_free_limit", +fun(Conf) -> + Settings = cuttlefish_variable:filter_by_prefix("disk_free_limit", Conf), + Absolute = proplists:get_value(["disk_free_limit", "absolute"], Settings), + Relative = proplists:get_value(["disk_free_limit", "relative"], Settings), + case {Absolute, Relative} of + {undefined, undefined} -> cuttlefish:invalid("No disk limit defined"); + {_, undefined} -> Absolute; + _ -> {mem_relative, Relative} + end +end}. + +%% +%% Clustering +%% ===================== +%% + +%% How to respond to cluster partitions. +%% See http://www.rabbitmq.com/partitions.html for further details. +%% +%% {cluster_partition_handling, ignore}, + +{mapping, "cluster_partition_handling", "rabbit.cluster_partition_handling", + [{datatype, {enum, [ignore, pause_minority, autoheal, pause_if_all_down]}}]}. + +{mapping, "cluster_partition_handling.pause_if_all_down.recover", + "rabbit.cluster_partition_handling", + [{datatype, {enum, [ignore, autoheal]}}]}. + +{mapping, "cluster_partition_handling.pause_if_all_down.nodes.$name", + "rabbit.cluster_partition_handling", + [{datatype, atom}]}. + +{translation, "rabbit.cluster_partition_handling", +fun(Conf) -> + case cuttlefish:conf_get("cluster_partition_handling", Conf) of + pause_if_all_down -> + PauseIfAllDownNodes = cuttlefish_variable:filter_by_prefix( + "cluster_partition_handling.pause_if_all_down.nodes", + Conf), + case PauseIfAllDownNodes of + [] -> + cuttlefish:invalid("Nodes required for pause_if_all_down"); + _ -> + Nodes = [ V || {K,V} <- PauseIfAllDownNodes ], + PauseIfAllDownRecover = cuttlefish:conf_get( + "cluster_partition_handling.pause_if_all_down.recover", + Conf), + case PauseIfAllDownRecover of + Recover when Recover == ignore; Recover == autoheal -> + {pause_if_all_down, Nodes, Recover}; + Invalid -> + cuttlefish:invalid("Recover strategy required for pause_if_all_down") + end + end; + Other -> Other + end +end}. + +%% Mirror sync batch size, in messages. Increasing this will speed +%% up syncing but total batch size in bytes must not exceed 2 GiB. +%% Available in RabbitMQ 3.6.0 or later. +%% +%% {mirroring_sync_batch_size, 4096}, + +{mapping, "mirroring_sync_batch_size", "rabbit.mirroring_sync_batch_size", + [{datatype, bytesize}, {validators, ["size_less_than_2G"]}]}. + +%% Make clustering happen *automatically* at startup - only applied +%% to nodes that have just been reset or started for the first time. +%% See http://www.rabbitmq.com/clustering.html#auto-config for +%% further details. +%% +%% {cluster_nodes, {['rabbit@my.host.com'], disc}}, + +{mapping, "autocluster.classic_config.nodes.$node", "rabbit.cluster_nodes", + [{datatype, atom}]}. + +{mapping, "autocluster.classic_config.node_type", "rabbit.cluster_nodes", [ + {datatype, {enum, [disc, disk, ram]}}, + {default, disc} +]}. + +{translation, "rabbit.cluster_nodes", +fun(Conf) -> + Nodes = [V || {_, V} <- cuttlefish_variable:filter_by_prefix("autocluster.classic_config.nodes", Conf)], + + case Nodes of + [] -> cuttlefish:unset(); + Other -> + case cuttlefish:conf_get("autocluster.classic_config.node_type", Conf, disc) of + disc -> {Other, disc}; + % Always cast to `disc` + disk -> {Other, disc}; + ram -> {Other, ram} + end + end +end}. + + +%% Interval (in milliseconds) at which we send keepalive messages +%% to other cluster members. Note that this is not the same thing +%% as net_ticktime; missed keepalive messages will not cause nodes +%% to be considered down. +%% +%% {cluster_keepalive_interval, 10000}, + +{mapping, "cluster_keepalive_interval", "rabbit.cluster_keepalive_interval", + [{datatype, integer}]}. + + +{mapping, "queue_master_locator", "rabbit.queue_master_locator", + [{datatype, string}]}. + +{translation, "rabbit.queue_master_locator", +fun(Conf) -> + list_to_binary(cuttlefish:conf_get("queue_master_locator", Conf)) +end}. + +%% +%% Statistics Collection +%% ===================== +%% + +%% Set (internal) statistics collection granularity. +%% +%% {collect_statistics, none}, + +{mapping, "collect_statistics", "rabbit.collect_statistics", + [{datatype, {enum, [none, coarse, fine]}}]}. + +%% Statistics collection interval (in milliseconds). Increasing +%% this will reduce the load on management database. +%% +%% {collect_statistics_interval, 5000}, + +{mapping, "collect_statistics_interval", "rabbit.collect_statistics_interval", + [{datatype, integer}]}. + +%% +%% Misc/Advanced Options +%% ===================== +%% +%% NB: Change these only if you understand what you are doing! +%% + +%% Explicitly enable/disable hipe compilation. +%% +%% {hipe_compile, true}, + +{mapping, "hipe_compile", "rabbit.hipe_compile", + [{datatype, {enum, [true, false]}}]}. + +%% Timeout used when waiting for Mnesia tables in a cluster to +%% become available. +%% +%% {mnesia_table_loading_timeout, 30000}, + +{mapping, "mnesia_table_loading_timeout", "rabbit.mnesia_table_loading_timeout", + [{datatype, integer}]}. + +%% Size in bytes below which to embed messages in the queue index. See +%% http://www.rabbitmq.com/persistence-conf.html +%% +%% {queue_index_embed_msgs_below, 4096} + +{mapping, "queue_index_embed_msgs_below", "rabbit.queue_index_embed_msgs_below", + [{datatype, bytesize}]}. + +% ========================== +% Lager section +% ========================== + +{mapping, "log.dir", "lager.log_root", [ + {datatype, string}, + {validators, ["dir_writable"]}]}. + +{mapping, "log.console", "lager.handlers", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "log.syslog", "lager.handlers", [ + {datatype, {enum, [true, false]}} +]}. +{mapping, "log.file", "lager.handlers", [ + {datatype, [{enum, [false]}, string]} +]}. + +{mapping, "log.file.level", "lager.handlers", [ + {datatype, {enum, [debug, info, warning, error]}} +]}. +{mapping, "log.$handler.level", "lager.handlers", [ + {datatype, {enum, [debug, info, warning, error]}} +]}. +{mapping, "log.file.rotation.date", "lager.handlers", [ + {datatype, string} +]}. +{mapping, "log.file.rotation.size", "lager.handlers", [ + {datatype, integer} +]}. +{mapping, "log.file.rotation.count", "lager.handlers", [ + {datatype, integer} +]}. + +{mapping, "log.syslog.identity", "lager.handlers", [ + {datatype, string} +]}. +{mapping, "log.syslog.facility", "lager.handlers", [ + {datatype, atom} +]}. + +{translation, "lager.handlers", +fun(Conf) -> + ConsoleHandler = case cuttlefish:conf_get("log.console", Conf, false) of + true -> + ConsoleLevel = cuttlefish:conf_get("log.console.level", Conf, info), + [{lager_console_backend, ConsoleLevel}]; + false -> [] + end, + FileHandler = case cuttlefish:conf_get("log.file", Conf, false) of + false -> []; + File -> + FileLevel = cuttlefish:conf_get("log.file.level", Conf, info), + RotationDate = cuttlefish:conf_get("log.file.rotation.date", Conf, ""), + RotationSize = cuttlefish:conf_get("log.file.rotation.size", Conf, 0), + RotationCount = cuttlefish:conf_get("log.file.rotation.count", Conf, 10), + [{lager_file_backend, [{file, File}, + {level, FileLevel}, + {date, RotationDate}, + {size, RotationSize}, + {count, RotationCount}]}] + end, + SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf, false) of + false -> []; + true -> + SyslogLevel = cuttlefish:conf_get("log.syslog.level", Conf, info), + Identity = cuttlefish:conf_get("log.syslog.identity", Conf), + Facility = cuttlefish:conf_get("log.syslog.facility", Conf), + [{lager_syslog_backend, [Identity, Facility, SyslogLevel]}] + end, + case ConsoleHandler ++ FileHandler ++ SyslogHandler of + [] -> undefined; + Other -> Other + end +end}. + + +% =============================== +% Validators +% =============================== + +{validator, "size_less_than_2G", "Byte size should be less than 2G and greater than 0", +fun(Size) when is_integer(Size) -> + Size > 0 andalso Size < 2147483648 +end}. + +{validator, "less_than_1", "Flooat is not beetween 0 and 1", +fun(Float) when is_float(Float) -> + Float > 0 andalso Float < 1 +end}. + +{validator, "port", "Invalid port number", +fun(Port) when is_integer(Port) -> + Port > 0 andalso Port < 65535 +end}. + +{validator, "byte", "Integer is not 0<i<255", +fun(Int) when is_integer(Int) -> + Int > 0 andalso Int < 255 +end}. + +{validator, "dir_writable", "Cannot create file in dir", +fun(Dir) -> + TestFile = filename:join(Dir, "test_file"), + file:delete(TestFile), + Res = ok == file:write_file(TestFile, <<"test">>), + file:delete(TestFile), + Res +end}. + +{validator, "file_accessible", "file doesnt exist or unaccessible", +fun(File) -> + ReadFile = file:read_file_info(File), + element(1, ReadFile) == ok +end}. + +{validator, "is_ip", "string is a valid IP address", +fun(IpStr) -> + Res = inet:parse_address(IpStr), + element(1, Res) == ok +end}. + +{validator, "non_negative_integer", "number should be greater or equal to zero", +fun(Int) when is_integer(Int) -> + Int >= 0 +end}. diff --git a/rabbitmq.conf.d/ldap.conf b/rabbitmq.conf.d/ldap.conf new file mode 100644 index 0000000000..2f51cbb409 --- /dev/null +++ b/rabbitmq.conf.d/ldap.conf @@ -0,0 +1,138 @@ +# ## ---------------------------------------------------------------------------- +# ## RabbitMQ LDAP Plugin +# ## +# ## See http://www.rabbitmq.com/ldap.html for details. +# ## +# ## ---------------------------------------------------------------------------- + + +# ======================================= +# LDAP section +# ======================================= + +# Should be defined in additional.conf maybe? + +# {rabbitmq_auth_backend_ldap, +# [## +# ## Connecting to the LDAP server(s) +# ## ================================ +# ## + +# ## Specify servers to bind to. You *must* set this in order for the plugin +# ## to work properly. +# ## +# ## {servers, ["your-server-name-goes-here"]}, + +ldap.servers.myserver = your-server-name-goes-here + +# ## Connect to the LDAP server using SSL +# ## +# ## {use_ssl, false}, + +ldap.use_ssl = false + +# ## Specify the LDAP port to connect to +# ## +# ## {port, 389}, + +ldap.port = 389 + +# ## LDAP connection timeout, in milliseconds or 'infinity' +# ## +# ## {timeout, infinity}, + +ldap.timeout = infinity + +# Or number +# ldap.timeout = 500 + +# ## Enable logging of LDAP queries. +# ## One of +# ## - false (no logging is performed) +# ## - true (verbose logging of the logic used by the plugin) +# ## - network (as true, but additionally logs LDAP network traffic) +# ## +# ## Defaults to false. +# ## +# ## {log, false}, + +ldap.log = false + +# Also can be true or network +# ldap.log = true +# ldap.log = network + +# ## +# ## Authentication +# ## ============== +# ## + +# ## Pattern to convert the username given through AMQP to a DN before +# ## binding +# ## +# ## {user_dn_pattern, "cn=${username},ou=People,dc=example,dc=com"}, + +ldap.user_dn_pattern = cn=${username},ou=People,dc=example,dc=com + +# ## Alternatively, you can convert a username to a Distinguished +# ## Name via an LDAP lookup after binding. See the documentation for +# ## full details. + +# ## When converting a username to a dn via a lookup, set these to +# ## the name of the attribute that represents the user name, and the +# ## base DN for the lookup query. +# ## +# ## {dn_lookup_attribute, "userPrincipalName"}, +# ## {dn_lookup_base, "DC=gopivotal,DC=com"}, + +ldap.dn_lookup_attribute = userPrincipalName +ldap.dn_lookup_base = DC=gopivotal,DC=com + +# ## Controls how to bind for authorisation queries and also to +# ## retrieve the details of users logging in without presenting a +# ## password (e.g., SASL EXTERNAL). +# ## One of +# ## - as_user (to bind as the authenticated user - requires a password) +# ## - anon (to bind anonymously) +# ## - {UserDN, Password} (to bind with a specified user name and password) +# ## +# ## Defaults to 'as_user'. +# ## +# ## {other_bind, as_user}, + +ldap.other_bind = as_user + +# Or can be more complex: +# ldap.other_bind.user_dn = User +# ldap.other_bind.password = Password +# If user_dn and password defined - other options is ignored. + +# ----------------------------- +# Too complex section of LDAP +# ----------------------------- + +# ## +# ## Authorisation +# ## ============= +# ## + +# ## The LDAP plugin can perform a variety of queries against your +# ## LDAP server to determine questions of authorisation. See +# ## http://www.rabbitmq.com/ldap.html#authorisation for more +# ## information. + +# ## Set the query to use when determining vhost access +# ## +# ## {vhost_access_query, {in_group, +# ## "ou=${vhost}-users,ou=vhosts,dc=example,dc=com"}}, + +# ## Set the query to use when determining resource (e.g., queue) access +# ## +# ## {resource_access_query, {constant, true}}, + +# ## Set queries to determine which tags a user has +# ## +# ## {tag_queries, []} +# ]}, +# ----------------------------- + diff --git a/rabbitmq.conf.d/rabbitmq.conf b/rabbitmq.conf.d/rabbitmq.conf new file mode 100644 index 0000000000..e702ec08b4 --- /dev/null +++ b/rabbitmq.conf.d/rabbitmq.conf @@ -0,0 +1,726 @@ +# ====================================== +# RabbbitMQ broker section +# ====================================== + +## Network Connectivity +## ==================== +## +## By default, RabbitMQ will listen on all interfaces, using +## the standard (reserved) AMQP port. +## +listener.tcp.default = 5672 + + +## To listen on a specific interface, provide an IP address with port. +## For example, to listen only on localhost for both IPv4 and IPv6: +## +# IPv4 +# listener.tcp.local = 127.0.0.1:5672 +# IPv6 +# listener.tcp.local_v6 = ::1:5672 + +## You can define multiple listeners using listener names +# listener.tcp.other_port = 5673 +# listener.tcp.other_ip = 10.10.10.10:5672 + + +## SSL listeners are configured in the same fashion as TCP listeners, +## including the option to control the choice of interface. +## +# listener.ssl.default = 5671 + +## Number of Erlang processes that will accept connections for the TCP +## and SSL listeners. +## +num_acceptors.tcp = 10 +num_acceptors.ssl = 1 + + +## Maximum time for AMQP 0-8/0-9/0-9-1 handshake (after socket connection +## and SSL handshake), in milliseconds. +## +handshake_timeout = 10000 + +## Set to 'true' to perform reverse DNS lookups when accepting a +## connection. Hostnames will then be shown instead of IP addresses +## in rabbitmqctl and the management plugin. +## +reverse_dns_lookups = true + +## +## Security / AAA +## ============== +## + +## The default "guest" user is only permitted to access the server +## via a loopback interface (e.g. localhost). +## {loopback_users, [<<"guest">>]}, +## +loopback_user.guest = true + +## Uncomment the following line if you want to allow access to the +## guest user from anywhere on the network. +# loopback_user.guest = false + +## Configuring SSL. +## See http://www.rabbitmq.com/ssl.html for full documentation. +## +ssl_option.verify = verify_peer +ssl_option.fail_if_no_peer_cert = false +# ssl_option.cacertfile = /path/to/rabbitmq.crt +# ssl_option.certfile = /path/to/rabbitmq.crt +# ssl_option.keyfile = /path/to/rabbitmq.key + +## Choose the available SASL mechanism(s) to expose. +## The two default (built in) mechanisms are 'PLAIN' and +## 'AMQPLAIN'. Additional mechanisms can be added via +## plugins. +## +## See http://www.rabbitmq.com/authentication.html for more details. +## +auth_mechanism.plain = PLAIN +auth_mechanism.amqplain = AMQPLAIN + +## Select an authentication database to use. RabbitMQ comes bundled +## with a built-in auth-database, based on mnesia. +## +auth_backends.1 = internal + +auth_backends.2.authn = ldap +auth_backends.2.authz = internal + +auth_backends.3.authz = rabbit_auth_backend_uaa + +## Configurations supporting the rabbitmq_auth_mechanism_ssl and +## rabbitmq_auth_backend_ldap plugins. +## +## NB: These options require that the relevant plugin is enabled. +## See http://www.rabbitmq.com/plugins.html for further details. + + +## The RabbitMQ-auth-mechanism-ssl plugin makes it possible to +## authenticate a user based on the client's SSL certificate. +## +## To use auth-mechanism-ssl, add to or replace the auth_mechanisms +## with EXTERNAL value. +## +#auth_mechanism.external = EXTERNAL + +## The rabbitmq_auth_backend_ldap plugin allows the broker to +## perform authentication and authorisation by deferring to an +## external LDAP server. +## +## For more information about configuring the LDAP backend, see +## http://www.rabbitmq.com/ldap.html. +## +## Enable the LDAP auth backend by adding to or replacing the +## auth_backends entry: +## +# auth_backends.2 = rabbit_auth_backend_ldap + +## Add another backend +# auth_backend.3 = rabbit_auth_backend_http + + +## This pertains to both the rabbitmq_auth_mechanism_ssl plugin and +## STOMP ssl_cert_login configurations. See the rabbitmq_stomp +## configuration section later in this file and the README in +## https://github.com/rabbitmq/rabbitmq-auth-mechanism-ssl for further +## details. +## +## To use the SSL cert's CN instead of its DN as the username +## +# ssl_cert_login_from = common_name + +## SSL handshake timeout, in milliseconds. +## +# ssl_handshake_timeout = 5000 + + +## Password hashing implementation. Will only affect newly +## created users. To recalculate hash for an existing user +## it's necessary to update her password. +## +## To use SHA-512, set to rabbit_password_hashing_sha512. +## +password_hashing_module = rabbit_password_hashing_sha256 + +## When importing definitions exported from versions earlier +## than 3.6.0, it is possible to go back to MD5 (only do this +## as a temporary measure!) by setting this to rabbit_password_hashing_md5. +## +# password_hashing_module = rabbit_password_hashing_md5 + +## +## Default User / VHost +## ==================== +## + +## On first start RabbitMQ will create a vhost and a user. These +## config items control what gets created. See +## http://www.rabbitmq.com/access-control.html for further +## information about vhosts and access control. +## +default_vhost = / +default_user = guest +default_pass = guest + +default_permissions.configure = .* +default_permissions.read = .* +default_permissions.write = .* + +## Tags for default user +## +## For more details about tags, see the documentation for the +## Management Plugin at http://www.rabbitmq.com/management.html. +## +default_user_tags.administrator = true + +## Define other tags like this: +# default_user_tags.management = true +# default_user_tags.custom_tag = true + +## +## Additional network and protocol related configuration +## ===================================================== +## + +## Set the default AMQP heartbeat delay (in seconds). +## +heartbeat = 600 + +## Set the max permissible size of an AMQP frame (in bytes). +## +frame_max = 131072 + +## Set the max frame size the server will accept before connection +## tuning occurs +## +initial_frame_max = 4096 + +## Set the max permissible number of channels per connection. +## 0 means "no limit". +## +channel_max = 128 + +## Customising Socket Options. +## +## See (http://www.erlang.org/doc/man/inet.html#setopts-2) for +## further documentation. +## + +tcp_listen_option.backlog = 128 +tcp_listen_option.nodelay = true +tcp_listen_option.exit_on_close = false + +## +## Resource Limits & Flow Control +## ============================== +## +## See http://www.rabbitmq.com/memory.html for full details. + +## Memory-based Flow Control threshold. +## +vm_memory_high_watermark.relative = 0.4 + +## Alternatively, we can set a limit (in bytes) of RAM used by the node. +## +# vm_memory_high_watermark.absolute = 1073741824 + +## Or you can set absolute value using memory units (with RabbitMQ 3.6.0+). +## Absolute watermark will be ignored if relative is defined! +## +# vm_memory_high_watermark.absolute = 2GB +## +## Supported units suffixes: +## +## kb, KB: kibibytes (2^10 bytes) +## mb, MB: mebibytes (2^20) +## gb, GB: gibibytes (2^30) + + + +## Fraction of the high watermark limit at which queues start to +## page message out to disc in order to free up memory. +## +## Values greater than 0.9 can be dangerous and should be used carefully. +## +vm_memory_high_watermark_paging_ratio = 0.5 + +## Interval (in milliseconds) at which we perform the check of the memory +## levels against the watermarks. +## +memory_monitor_interval = 2500 + +## Set disk free limit (in bytes). Once free disk space reaches this +## lower bound, a disk alarm will be set - see the documentation +## listed above for more details. +## +## Absolute watermark will be ignored if relative is defined! +disk_free_limit.absolute = 50000 + +## Or you can set it using memory units (same as in vm_memory_high_watermark) +## with RabbitMQ 3.6.0+. +# disk_free_limit.absolute = 500KB +# disk_free_limit.absolute = 50mb +# disk_free_limit.absolute = 5GB + +## Alternatively, we can set a limit relative to total available RAM. +## +## Values lower than 1.0 can be dangerous and should be used carefully. +disk_free_limit.relative = 2.0 + +## +## Clustering +## ===================== +## +cluster_partition_handling = ignore + +## pause_if_all_down strategy require additional configuration +# cluster_partition_handling = pause_if_all_down + +## Recover strategy. Can be either 'autoheal' or 'ignore' +# cluster_partition_handling.pause_if_all_down.recover = ignore + +## Node names to check +# cluster_partition_handling.pause_if_all_down.node.rabbit = rabbit@localhost +# cluster_partition_handling.pause_if_all_down.node.hare = hare@localhost + +## Mirror sync batch size, in messages. Increasing this will speed +## up syncing but total batch size in bytes must not exceed 2 GiB. +## Available in RabbitMQ 3.6.0 or later. +## +mirroring_sync_batch_size = 4096 + +## Make clustering happen *automatically* at startup - only applied +## to nodes that have just been reset or started for the first time. +## See http://www.rabbitmq.com/clustering.html#auto-config for +## further details. +## +# cluster_nodes.disc.1 = rabbit@my.host.com + +## You can define multiple nodes +# cluster_nodes.disc.2 = hare@my.host.com + +## There can be also ram nodes. +## Ram nodes should not be defined together with disk nodes +# cluster_nodes.ram.1 = rabbit@my.host.com + +## Interval (in milliseconds) at which we send keepalive messages +## to other cluster members. Note that this is not the same thing +## as net_ticktime; missed keepalive messages will not cause nodes +## to be considered down. +## +# cluster_keepalive_interval = 10000 + +## +## Statistics Collection +## ===================== +## + +## Set (internal) statistics collection granularity. +## +## Can be none, coarse or fine +collect_statistics = none + +# collect_statistics = coarse + +## Statistics collection interval (in milliseconds). Increasing +## this will reduce the load on management database. +## +collect_statistics_interval = 5000 + +## +## Misc/Advanced Options +## ===================== +## +## NB: Change these only if you understand what you are doing! +## + +## Explicitly enable/disable hipe compilation. +## +hipe_compile = false + +## Timeout used when waiting for Mnesia tables in a cluster to +## become available. +## +mnesia_table_loading_timeout = 30000 + +## Size in bytes below which to embed messages in the queue index. See +## http://www.rabbitmq.com/persistence-conf.html +## +queue_index_embed_msgs_below = 4096 + +## You can also set this size in memory units +## +queue_index_embed_msgs_below = 4kb + +## ---------------------------------------------------------------------------- +## Advanced Erlang Networking/Clustering Options. +## +## See http://www.rabbitmq.com/clustering.html for details +## ---------------------------------------------------------------------------- + +# ====================================== +# Kernel section +# ====================================== + +# kernel.net_ticktime = 60 + +## ---------------------------------------------------------------------------- +## RabbitMQ Management Plugin +## +## See http://www.rabbitmq.com/management.html for details +## ---------------------------------------------------------------------------- + +# ======================================= +# Management section +# ======================================= + +## Pre-Load schema definitions from the following JSON file. See +## http://www.rabbitmq.com/management.html#load-definitions +## +# management.load_definitions = /path/to/schema.json + +## Log all requests to the management HTTP API to a file. +## +# management.http_log_dir = /path/to/access.log + +## Change the port on which the HTTP listener listens, +## specifying an interface for the web server to bind to. +## Also set the listener to use SSL and provide SSL options. +## + +# QA: Maybe use IP type like in tcp_listener? +management.listener.port = 12345 +management.listener.ip = 127.0.0.1 +# management.listener.ssl = true + +# management.listener.ssl_opts.cacertfile = /path/to/cacert.pem +# management.listener.ssl_opts.certfile = /path/to/cert.pem +# management.listener.ssl_opts.keyfile = /path/to/key.pem + +## One of 'basic', 'detailed' or 'none'. See +## http://www.rabbitmq.com/management.html#fine-stats for more details. +management.rates_mode = basic + +## Configure how long aggregated data (such as message rates and queue +## lengths) is retained. Please read the plugin's documentation in +## http://www.rabbitmq.com/management.html#configuration for more +## details. +## Your can use 'minute', 'hour' and '24hours' keys or integer key (in seconds) +management.sample_retention_policies.global.minute = 5 +management.sample_retention_policies.global.hour = 60 +management.sample_retention_policies.global.day = 1200 + +management.sample_retention_policies.basic.minute = 5 +management.sample_retention_policies.basic.hour = 60 + +management.sample_retention_policies.detailed.10 = 5 + +## ---------------------------------------------------------------------------- +## RabbitMQ Shovel Plugin +## +## See http://www.rabbitmq.com/shovel.html for details +## ---------------------------------------------------------------------------- + +## Shovel plugin config example is defined in additional.config file + + +## ---------------------------------------------------------------------------- +## RabbitMQ Stomp Adapter +## +## See http://www.rabbitmq.com/stomp.html for details +## ---------------------------------------------------------------------------- + +# ======================================= +# STOMP section +# ======================================= + +## Network Configuration - the format is generally the same as for the broker +## +stomp.listener.tcp.default = 61613 + +## Same for ssl listeners +## +# stomp.listener.ssl.default = 61614 + +## Number of Erlang processes that will accept connections for the TCP +## and SSL listeners. +## +stomp.num_acceptors.tcp = 10 +stomp.num_acceptors.ssl = 1 + +## Additional SSL options + +## Extract a name from the client's certificate when using SSL. +## +stomp.ssl_cert_login = true + +## Set a default user name and password. This is used as the default login +## whenever a CONNECT frame omits the login and passcode headers. +## +## Please note that setting this will allow clients to connect without +## authenticating! +## +# stomp.default_user = guest +# stomp.default_pass = guest + +## If a default user is configured, or you have configured use SSL client +## certificate based authentication, you can choose to allow clients to +## omit the CONNECT frame entirely. If set to true, the client is +## automatically connected as the default user or user supplied in the +## SSL certificate whenever the first frame sent on a session is not a +## CONNECT frame. +## +# stomp.implicit_connect = true + +## ---------------------------------------------------------------------------- +## RabbitMQ MQTT Adapter +## +## See https://github.com/rabbitmq/rabbitmq-mqtt/blob/stable/README.md +## for details +## ---------------------------------------------------------------------------- + +# ======================================= +# MQTT section +# ======================================= + +## Set the default user name and password. Will be used as the default login +## if a connecting client provides no other login details. +## +## Please note that setting this will allow clients to connect without +## authenticating! +## +# mqtt.default_user = guest +# mqtt.default_pass = guest + +## Enable anonymous access. If this is set to false, clients MUST provide +## login information in order to connect. See the default_user/default_pass +## configuration elements for managing logins without authentication. +## +# mqtt.allow_anonymous = true + +## If you have multiple chosts, specify the one to which the +## adapter connects. +## +mqtt.vhost = / + +## Specify the exchange to which messages from MQTT clients are published. +## +mqtt.exchange = amq.topic + +## Specify TTL (time to live) to control the lifetime of non-clean sessions. +## +# mqtt.subscription_ttl = 1800000 + +## Set the prefetch count (governing the maximum number of unacknowledged +## messages that will be delivered). +## +mqtt.prefetch = 10 + +## TCP/SSL Configuration (as per the broker configuration). +## +mqtt.listener.tcp.default = 1883 + +## Same for ssl listener +## +# mqtt.listener.ssl.default = 1884 + +## Number of Erlang processes that will accept connections for the TCP +## and SSL listeners. +## +mqtt.num_acceptors.tcp = 10 +mqtt.num_acceptors.ssl = 1 + +## TCP/Socket options (as per the broker configuration). +## +# mqtt.tcp_listen_option.backlog = 128 +# mqtt.tcp_listen_option.nodelay = true + +## ---------------------------------------------------------------------------- +## RabbitMQ AMQP 1.0 Support +## +## See https://github.com/rabbitmq/rabbitmq-amqp1.0/blob/stable/README.md +## for details +## ---------------------------------------------------------------------------- + +# ======================================= +# AMQP_1 section +# ======================================= + + +## Connections that are not authenticated with SASL will connect as this +## account. See the README for more information. +## +## Please note that setting this will allow clients to connect without +## authenticating! +## +amqp1_0.default_user = guest + +## Enable protocol strict mode. See the README for more information. +## +amqp1_0.protocol_strict_mode = false + +## Lager controls logging. +## See https://github.com/basho/lager for more documentation +## +## Log direcrory, taken from the RABBITMQ_LOG_BASE env variable by default. +## +# log.dir = /var/log/rabbitmq + +## Logging to console (can be true or false) +## +# log.console = false + +## Loglevel to log to console +## +# log.console.level = info + +## Logging to file. Can be false or filename. +## Default: +# log.file = rabbit.log + +## To turn off: +# log.file = false + +## Loglevel to log to file +## +# log.file.level = info + +## File rotation config. No rotation by defualt. +## DO NOT SET rotation date to ''. Leave unset if require "" value +# log.file.rotation.date = $D0 +# log.file.rotation.size = 0 + + +## QA: Config for syslog logging +# log.syslog = false +# log.syslog.identity = rabbitmq +# log.syslog.level = info +# log.syslog.facility = daemon + + +## ---------------------------------------------------------------------------- +## RabbitMQ LDAP Plugin +## +## See http://www.rabbitmq.com/ldap.html for details. +## +## ---------------------------------------------------------------------------- + +# ======================================= +# LDAP section +# ======================================= + +## +## Connecting to the LDAP server(s) +## ================================ +## + +## Specify servers to bind to. You *must* set this in order for the plugin +## to work properly. +## +# ldap.servers.1 = your-server-name-goes-here + +## You can define multiple servers +# ldap.servers.2 = your-other-server + +## Connect to the LDAP server using SSL +## +# ldap.use_ssl = false + +## Specify the LDAP port to connect to +## +# ldap.port = 389 + +## LDAP connection timeout, in milliseconds or 'infinity' +## +# ldap.timeout = infinity + +## Or number +# ldap.timeout = 500 + +## Enable logging of LDAP queries. +## One of +## - false (no logging is performed) +## - true (verbose logging of the logic used by the plugin) +## - network (as true, but additionally logs LDAP network traffic) +## +## Defaults to false. +## +# ldap.log = false + +## Also can be true or network +# ldap.log = true +# ldap.log = network + +## +## Authentication +## ============== +## + +## Pattern to convert the username given through AMQP to a DN before +## binding +## +# ldap.user_dn_pattern = cn=${username},ou=People,dc=example,dc=com + +## Alternatively, you can convert a username to a Distinguished +## Name via an LDAP lookup after binding. See the documentation for +## full details. + +## When converting a username to a dn via a lookup, set these to +## the name of the attribute that represents the user name, and the +## base DN for the lookup query. +## +# ldap.dn_lookup_attribute = userPrincipalName +# ldap.dn_lookup_base = DC=gopivotal,DC=com + +## Controls how to bind for authorisation queries and also to +## retrieve the details of users logging in without presenting a +## password (e.g., SASL EXTERNAL). +## One of +## - as_user (to bind as the authenticated user - requires a password) +## - anon (to bind anonymously) +## - {UserDN, Password} (to bind with a specified user name and password) +## +## Defaults to 'as_user'. +## +# ldap.other_bind = as_user + +## Or can be more complex: +# ldap.other_bind.user_dn = User +# ldap.other_bind.password = Password + +## If user_dn and password defined - other options is ignored. + +# ----------------------------- +# Too complex section of LDAP +# ----------------------------- + +## +## Authorisation +## ============= +## + +## The LDAP plugin can perform a variety of queries against your +## LDAP server to determine questions of authorisation. See +## http://www.rabbitmq.com/ldap.html#authorisation for more +## information. + +## Following configuration should be defined in additional.config file +## DO NOT UNCOMMENT THIS LINES! + +## Set the query to use when determining vhost access +## +## {vhost_access_query, {in_group, +## "ou=${vhost}-users,ou=vhosts,dc=example,dc=com"}}, + +## Set the query to use when determining resource (e.g., queue) access +## +## {resource_access_query, {constant, true}}, + +## Set queries to determine which tags a user has +## +## {tag_queries, []} +# ]}, +# ----------------------------- diff --git a/scripts/cuttlefish b/scripts/cuttlefish Binary files differnew file mode 100755 index 0000000000..6c1e4bbb89 --- /dev/null +++ b/scripts/cuttlefish diff --git a/scripts/rabbitmq-defaults b/scripts/rabbitmq-defaults index baffce80de..5342978694 100755 --- a/scripts/rabbitmq-defaults +++ b/scripts/rabbitmq-defaults @@ -38,6 +38,9 @@ CONFIG_FILE=${SYS_PREFIX}/etc/rabbitmq/rabbitmq LOG_BASE=${SYS_PREFIX}/var/log/rabbitmq MNESIA_BASE=${SYS_PREFIX}/var/lib/rabbitmq/mnesia ENABLED_PLUGINS_FILE=${SYS_PREFIX}/etc/rabbitmq/enabled_plugins +GENERATED_CONFIG_DIR=${SYS_PREFIX}/var/lib/rabbitmq/config +ADVANCED_CONFIG_FILE=${SYS_PREFIX}/etc/rabbitmq/advanced +SCHEMA_DIR=${SYS_PREFIX}/var/lib/rabbitmq/schema PLUGINS_DIR="${RABBITMQ_HOME}/plugins" diff --git a/scripts/rabbitmq-defaults.bat b/scripts/rabbitmq-defaults.bat index 8fff5ea827..0246dc64fd 100644 --- a/scripts/rabbitmq-defaults.bat +++ b/scripts/rabbitmq-defaults.bat @@ -41,6 +41,9 @@ set CONFIG_FILE=!RABBITMQ_BASE!\rabbitmq set LOG_BASE=!RABBITMQ_BASE!\log
set MNESIA_BASE=!RABBITMQ_BASE!\db
set ENABLED_PLUGINS_FILE=!RABBITMQ_BASE!\enabled_plugins
+set GENERATED_CONFIG_DIR=!RABBITMQ_BASE!\config
+set ADVANCED_CONFIG_FILE=!RABBITMQ_BASE!\advanced
+set SCHEMA_DIR=!RABBITMQ_BASE!\schema
REM PLUGINS_DIR="${RABBITMQ_HOME}/plugins"
for /f "delims=" %%F in ("!TDP0!..\plugins") do set PLUGINS_DIR=%%~dpsF%%~nF%%~xF
diff --git a/scripts/rabbitmq-env b/scripts/rabbitmq-env index 8c33e7c0b7..206bdd0c20 100755 --- a/scripts/rabbitmq-env +++ b/scripts/rabbitmq-env @@ -186,6 +186,9 @@ DEFAULT_NODE_PORT=5672 [ "x" = "x$RABBITMQ_SERVER_CODE_PATH" ] && RABBITMQ_SERVER_CODE_PATH=${SERVER_CODE_PATH} [ "x" = "x$RABBITMQ_MNESIA_DIR" ] && RABBITMQ_MNESIA_DIR=${MNESIA_DIR} [ "x" = "x$RABBITMQ_MNESIA_DIR" ] && RABBITMQ_MNESIA_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME} +[ "x" = "x$RABBITMQ_GENERATED_CONFIG_DIR" ] && RABBITMQ_GENERATED_CONFIG_DIR=${GENERATED_CONFIG_DIR} +[ "x" = "x$RABBITMQ_ADVANCED_CONFIG_FILE" ] && RABBITMQ_ADVANCED_CONFIG_FILE=${ADVANCED_CONFIG_FILE} +[ "x" = "x$RABBITMQ_SCHEMA_DIR" ] && RABBITMQ_SCHEMA_DIR=${SCHEMA_DIR} [ "x" = "x$RABBITMQ_IGNORE_SIGINT" ] && RABBITMQ_IGNORE_SIGINT="true" [ "xtrue" = "x$RABBITMQ_IGNORE_SIGINT" ] && RABBITMQ_IGNORE_SIGINT_FLAG="+B i" @@ -215,13 +218,11 @@ rmq_normalize_path_var RABBITMQ_PLUGINS_DIR ## Log rotation [ "x" = "x$RABBITMQ_LOGS" ] && RABBITMQ_LOGS=${LOGS} +[ "x" != "x$RABBITMQ_LOGS" ] && export RABBITMQ_LOGS_source=environment [ "x" = "x$RABBITMQ_LOGS" ] && RABBITMQ_LOGS="${RABBITMQ_LOG_BASE}/${RABBITMQ_NODENAME}.log" -[ "x" = "x$RABBITMQ_SASL_LOGS" ] && RABBITMQ_SASL_LOGS=${SASL_LOGS} -[ "x" = "x$RABBITMQ_SASL_LOGS" ] && RABBITMQ_SASL_LOGS="${RABBITMQ_LOG_BASE}/${RABBITMQ_NODENAME}-sasl.log" rmq_normalize_path_var \ - RABBITMQ_LOGS \ - RABBITMQ_SASL_LOGS + RABBITMQ_LOGS [ "x" = "x$RABBITMQ_CTL_ERL_ARGS" ] && RABBITMQ_CTL_ERL_ARGS=${CTL_ERL_ARGS} @@ -237,8 +238,7 @@ rmq_check_if_shared_with_mnesia \ RABBITMQ_PLUGINS_EXPAND_DIR \ RABBITMQ_ENABLED_PLUGINS_FILE \ RABBITMQ_PLUGINS_DIR \ - RABBITMQ_LOGS \ - RABBITMQ_SASL_LOGS + RABBITMQ_LOGS ##--- End of overridden <var_name> variables @@ -265,7 +265,8 @@ if [ "${RABBITMQ_DEV_ENV}" ]; then RABBITMQ_ENABLED_PLUGINS_FILE="${enabled_plugins_file}" fi fi - + + if [ -d "${RABBITMQ_PLUGINS_DIR}" ]; then # RabbitMQ was started with "make run-broker" from its own # source tree. Take rabbit_common from the plugins directory. diff --git a/scripts/rabbitmq-env.bat b/scripts/rabbitmq-env.bat index 3c84351d52..56b2f69b2d 100644 --- a/scripts/rabbitmq-env.bat +++ b/scripts/rabbitmq-env.bat @@ -92,8 +92,8 @@ if "!RABBITMQ_NODENAME!"=="" ( if "!NODENAME!"=="" (
REM We use Erlang to query the local hostname because
REM !COMPUTERNAME! and Erlang may return different results.
- REM Start erl with -sname to make sure epmd is started.
- call "%ERLANG_HOME%\bin\erl.exe" -A0 -noinput -boot start_clean -sname rabbit-prelaunch-epmd -eval "init:stop()." >nul 2>&1
+ REM Start erl with -sname to make sure epmd is started.
+ call "%ERLANG_HOME%\bin\erl.exe" -A0 -noinput -boot start_clean -sname rabbit-prelaunch-epmd -eval "init:stop()." >nul 2>&1
for /f "delims=" %%F in ('call "%ERLANG_HOME%\bin\erl.exe" -A0 -noinput -boot start_clean -eval "net_kernel:start([list_to_atom(""rabbit-gethostname-"" ++ os:getpid()), %NAMETYPE%]), [_, H] = string:tokens(atom_to_list(node()), ""@""), io:format(""~s~n"", [H]), init:stop()."') do @set HOSTNAME=%%F
set RABBITMQ_NODENAME=rabbit@!HOSTNAME!
set HOSTNAME=
@@ -167,6 +167,8 @@ if "!RABBITMQ_SERVER_ERL_ARGS!"=="" ( )
REM [ "x" = "x$RABBITMQ_CONFIG_FILE" ] && RABBITMQ_CONFIG_FILE=${CONFIG_FILE}
+
+CALL :unquote RABBITMQ_CONFIG_FILE %RABBITMQ_CONFIG_FILE%
if "!RABBITMQ_CONFIG_FILE!"=="" (
if "!CONFIG_FILE!"=="" (
set RABBITMQ_CONFIG_FILE=!RABBITMQ_BASE!\rabbitmq
@@ -175,6 +177,32 @@ if "!RABBITMQ_CONFIG_FILE!"=="" ( )
)
+if "!RABBITMQ_GENERATED_CONFIG_DIR!"=="" (
+ if "!GENERATED_CONFIG_DIR!"=="" (
+ set RABBITMQ_GENERATED_CONFIG_DIR=!RABBITMQ_BASE!\config
+ ) else (
+ set RABBITMQ_GENERATED_CONFIG_DIR=!GENERATED_CONFIG_DIR!
+ )
+)
+
+CALL :unquote RABBITMQ_ADVANCED_CONFIG_FILE %RABBITMQ_ADVANCED_CONFIG_FILE%
+if "!RABBITMQ_ADVANCED_CONFIG_FILE!"=="" (
+ if "!ADVANCED_CONFIG_FILE!"=="" (
+ set RABBITMQ_ADVANCED_CONFIG_FILE=!RABBITMQ_BASE!\advanced
+ ) else (
+ set RABBITMQ_ADVANCED_CONFIG_FILE=!ADVANCED_CONFIG_FILE!
+ )
+)
+
+if "!RABBITMQ_SCHEMA_DIR!" == "" (
+ if "!SCHEMA_DIR!"=="" (
+ set RABBITMQ_SCHEMA_DIR=!RABBITMQ_HOME!\priv\schema
+ ) else (
+ set RABBITMQ_SCHEMA_DIR=!SCHEMA_DIR!
+ )
+)
+
+
REM [ "x" = "x$RABBITMQ_LOG_BASE" ] && RABBITMQ_LOG_BASE=${LOG_BASE}
if "!RABBITMQ_LOG_BASE!"=="" (
if "!LOG_BASE!"=="" (
@@ -297,23 +325,6 @@ if not "!RABBITMQ_LOGS" == "-" ( for /f "delims=" %%F in ("!RABBITMQ_LOGS!") do set RABBITMQ_LOGS=%%~sF
)
-REM [ "x" = "x$RABBITMQ_SASL_LOGS" ] && RABBITMQ_SASL_LOGS=${SASL_LOGS}
-REM [ "x" = "x$RABBITMQ_SASL_LOGS" ] && RABBITMQ_SASL_LOGS="${RABBITMQ_LOG_BASE}/${RABBITMQ_NODENAME}-sasl.log"
-if "!RABBITMQ_SASL_LOGS!"=="" (
- if "!SASL_LOGS!"=="" (
- set RABBITMQ_SASL_LOGS=!RABBITMQ_LOG_BASE!\!RABBITMQ_NODENAME!-sasl.log
- ) else (
- set RABBITMQ_SASL_LOGS=!SASL_LOGS!
- )
-)
-if not "!RABBITMQ_SASL_LOGS" == "-" (
- if not exist "!RABBITMQ_SASL_LOGS!" (
- for /f "delims=" %%F in ("!RABBITMQ_SASL_LOGS!") do mkdir %%~dpF 2>NUL
- copy /y NUL "!RABBITMQ_SASL_LOGS!" >NUL
- )
- for /f "delims=" %%F in ("!RABBITMQ_SASL_LOGS!") do set RABBITMQ_SASL_LOGS=%%~sF
-)
-
REM [ "x" = "x$RABBITMQ_CTL_ERL_ARGS" ] && RABBITMQ_CTL_ERL_ARGS=${CTL_ERL_ARGS}
if "!$RABBITMQ_CTL_ERL_ARGS!"=="" (
if not "!CTL_ERL_ARGS!"=="" (
@@ -426,3 +437,7 @@ REM ##--- End of overridden <var_name> variables REM
REM # Since we source this elsewhere, don't accidentally stop execution
REM true
+
+:unquote
+set %1=%~2
+EXIT /B 0
diff --git a/scripts/rabbitmq-server b/scripts/rabbitmq-server index 7b0599e88f..48365252e5 100755 --- a/scripts/rabbitmq-server +++ b/scripts/rabbitmq-server @@ -62,6 +62,11 @@ RABBITMQ_EBIN_ROOT="${RABBITMQ_HOME}/ebin" set +e +RABBITMQ_ADVANCED_CONFIG_FILE_NOEX="${RABBITMQ_ADVANCED_CONFIG_FILE%.*}" +if [ "${RABBITMQ_ADVANCED_CONFIG_FILE_NOEX}.config" = "${RABBITMQ_ADVANCED_CONFIG_FILE}" ]; then + RABBITMQ_ADVANCED_CONFIG_FILE="${RABBITMQ_ADVANCED_CONFIG_FILE_NOEX}" +fi + # `net_kernel:start/1` will fail in `longnames` mode when erlang is # unable to determine FQDN of a node (with a dot in it). But `erl` # itself has some magic that still allow it to start when you @@ -84,6 +89,9 @@ RABBITMQ_DIST_PORT=$RABBITMQ_DIST_PORT \ -hidden \ -s rabbit_prelaunch \ ${RABBITMQ_NAME_TYPE} ${RABBITMQ_PRELAUNCH_NODENAME} \ + -conf_advanced "${RABBITMQ_ADVANCED_CONFIG_FILE}" \ + -rabbit enabled_plugins_file "\"$RABBITMQ_ENABLED_PLUGINS_FILE\"" \ + -rabbit plugins_dir "\"$RABBITMQ_PLUGINS_DIR\"" \ -extra "${RABBITMQ_NODENAME}" PRELAUNCH_RESULT=$? @@ -97,30 +105,60 @@ else exit ${PRELAUNCH_RESULT} fi +if [ ! -d ${RABBITMQ_SCHEMA_DIR} ]; then + mkdir "${RABBITMQ_SCHEMA_DIR}" +fi + +if [ ! -f "${RABBITMQ_SCHEMA_DIR}/rabbitmq.schema" ]; then + cp "${RABBITMQ_HOME}/priv/schema/rabbitmq.schema" "${RABBITMQ_SCHEMA_DIR}" +fi + set -e -RABBITMQ_CONFIG_ARG= -[ -f "${RABBITMQ_CONFIG_FILE}.config" ] && RABBITMQ_CONFIG_ARG="-config ${RABBITMQ_CONFIG_FILE}" +RABBITMQ_CONFIG_FILE_NOEX="${RABBITMQ_CONFIG_FILE%.*}" + +if [ "${RABBITMQ_CONFIG_FILE_NOEX}.config" = "${RABBITMQ_CONFIG_FILE}" ]; then + if [ -f "${RABBITMQ_CONFIG_FILE}" ]; then + RABBITMQ_CONFIG_ARG="-config ${RABBITMQ_CONFIG_FILE_NOEX}" + fi +elif [ "${RABBITMQ_CONFIG_FILE_NOEX}.conf" = "${RABBITMQ_CONFIG_FILE}" ]; then + RABBITMQ_CONFIG_ARG="-conf ${RABBITMQ_CONFIG_FILE_NOEX} \ + -conf_dir ${RABBITMQ_GENERATED_CONFIG_DIR} \ + -conf_script_dir `dirname $0` \ + -conf_schema_dir ${RABBITMQ_SCHEMA_DIR}" + if [ -f "${RABBITMQ_ADVANCED_CONFIG_FILE}.config" ]; then + RABBITMQ_CONFIG_ARG="${RABBITMQ_CONFIG_ARG} \ + -conf_advanced ${RABBITMQ_ADVANCED_CONFIG_FILE} \ + -config ${RABBITMQ_ADVANCED_CONFIG_FILE}" + fi +else + if [ -f "${RABBITMQ_CONFIG_FILE}.config" ]; then + RABBITMQ_CONFIG_ARG="-config ${RABBITMQ_CONFIG_FILE}" + elif [ -f "${RABBITMQ_CONFIG_FILE}.conf" ]; then + RABBITMQ_CONFIG_ARG="-conf ${RABBITMQ_CONFIG_FILE} \ + -conf_dir ${RABBITMQ_GENERATED_CONFIG_DIR} \ + -conf_script_dir `dirname $0` \ + -conf_schema_dir ${RABBITMQ_SCHEMA_DIR}" + if [ -f "${RABBITMQ_ADVANCED_CONFIG_FILE}.config" ]; then + RABBITMQ_CONFIG_ARG="${RABBITMQ_CONFIG_ARG} \ + -conf_advanced ${RABBITMQ_ADVANCED_CONFIG_FILE} \ + -config ${RABBITMQ_ADVANCED_CONFIG_FILE}" + fi + fi +fi RABBITMQ_LISTEN_ARG= [ "x" != "x$RABBITMQ_NODE_PORT" ] && [ "x" != "x$RABBITMQ_NODE_IP_ADDRESS" ] && RABBITMQ_LISTEN_ARG="-rabbit tcp_listeners [{\""${RABBITMQ_NODE_IP_ADDRESS}"\","${RABBITMQ_NODE_PORT}"}]" -# If $RABBITMQ_LOGS is '-', send all log messages to stdout. Likewise -# for RABBITMQ_SASL_LOGS. This is particularily useful for Docker -# images. +# If $RABBITMQ_LOGS is '-', send all log messages to stdout. This is +# particularily useful for Docker images. if [ "$RABBITMQ_LOGS" = '-' ]; then - RABBIT_ERROR_LOGGER='tty' -else - RABBIT_ERROR_LOGGER='{file,"'${RABBITMQ_LOGS}'"}' -fi - -if [ "$RABBITMQ_SASL_LOGS" = '-' ]; then SASL_ERROR_LOGGER=tty - RABBIT_SASL_ERROR_LOGGER='tty' + RABBIT_LAGER_HANDLER=tty else SASL_ERROR_LOGGER=false - RABBIT_SASL_ERROR_LOGGER='{file,"'${RABBITMQ_SASL_LOGS}'"}' + RABBIT_LAGER_HANDLER='"'${RABBITMQ_LOGS}'"' fi # we need to turn off path expansion because some of the vars, notably @@ -166,8 +204,8 @@ start_rabbitmq_server() { ${RABBITMQ_LISTEN_ARG} \ -sasl errlog_type error \ -sasl sasl_error_logger "$SASL_ERROR_LOGGER" \ - -rabbit error_logger "$RABBIT_ERROR_LOGGER" \ - -rabbit sasl_error_logger "$RABBIT_SASL_ERROR_LOGGER" \ + -rabbit lager_log_root "\"$RABBITMQ_LOG_BASE\"" \ + -rabbit lager_handler "$RABBIT_LAGER_HANDLER" \ -rabbit enabled_plugins_file "\"$RABBITMQ_ENABLED_PLUGINS_FILE\"" \ -rabbit plugins_dir "\"$RABBITMQ_PLUGINS_DIR\"" \ -rabbit plugins_expand_dir "\"$RABBITMQ_PLUGINS_EXPAND_DIR\"" \ diff --git a/scripts/rabbitmq-server.bat b/scripts/rabbitmq-server.bat index 585a830efa..a15f24e586 100644 --- a/scripts/rabbitmq-server.bat +++ b/scripts/rabbitmq-server.bat @@ -21,6 +21,7 @@ rem Preserve values that might contain exclamation marks before rem enabling delayed expansion
set TDP0=%~dp0
set STAR=%*
+set CONF_SCRIPT_DIR="%~dp0"
setlocal enabledelayedexpansion
REM Get default settings with user overrides for (RABBITMQ_)<var_name>
@@ -41,11 +42,19 @@ if not exist "!ERLANG_HOME!\bin\erl.exe" ( set RABBITMQ_EBIN_ROOT=!RABBITMQ_HOME!\ebin
+CALL :get_noex !RABBITMQ_ADVANCED_CONFIG_FILE! RABBITMQ_ADVANCED_CONFIG_FILE_NOEX
+if "!RABBITMQ_ADVANCED_CONFIG_FILE!" == "!RABBITMQ_ADVANCED_CONFIG_FILE_NOEX!.config" (
+ set RABBITMQ_ADVANCED_CONFIG_FILE=!RABBITMQ_ADVANCED_CONFIG_FILE_NOEX!
+)
+
"!ERLANG_HOME!\bin\erl.exe" ^
-pa "!RABBITMQ_EBIN_ROOT!" ^
-noinput -hidden ^
-s rabbit_prelaunch ^
!RABBITMQ_NAME_TYPE! rabbitmqprelaunch!RANDOM!!TIME:~9! ^
+ -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+ -rabbit enabled_plugins_file "!RABBITMQ_ENABLED_PLUGINS_FILE!" ^
+ -rabbit plugins_dir "!$RABBITMQ_PLUGINS_DIR!" ^
-extra "!RABBITMQ_NODENAME!"
if ERRORLEVEL 2 (
@@ -56,12 +65,46 @@ if ERRORLEVEL 2 ( set RABBITMQ_DIST_ARG=-kernel inet_dist_listen_min !RABBITMQ_DIST_PORT! -kernel inet_dist_listen_max !RABBITMQ_DIST_PORT!
)
+if not exist "!RABBITMQ_SCHEMA_DIR!" (
+ mkdir "!RABBITMQ_SCHEMA_DIR!"
+)
+
+if not exist "!RABBITMQ_SCHEMA_DIR!\rabbitmq.schema" (
+ copy "!RABBITMQ_HOME!\priv\schema\rabbitmq.schema" "!RABBITMQ_SCHEMA_DIR!\rabbitmq.schema"
+)
+
set RABBITMQ_EBIN_PATH="-pa !RABBITMQ_EBIN_ROOT!"
-if exist "!RABBITMQ_CONFIG_FILE!.config" (
- set RABBITMQ_CONFIG_ARG=-config "!RABBITMQ_CONFIG_FILE!"
+CALL :get_noex !RABBITMQ_CONFIG_FILE! RABBITMQ_CONFIG_FILE_NOEX
+
+if "!RABBITMQ_CONFIG_FILE!" == "!RABBITMQ_CONFIG_FILE_NOEX!.config" (
+ if exist "!RABBITMQ_CONFIG_FILE!" (
+ set RABBITMQ_CONFIG_ARG=-config "!RABBITMQ_CONFIG_FILE_NOEX!"
+ )
+) else if "!RABBITMQ_CONFIG_FILE!" == "!RABBITMQ_CONFIG_FILE_NOEX!.conf" (
+ set RABBITMQ_CONFIG_ARG=-conf "!RABBITMQ_CONFIG_FILE_NOEX!" ^
+ -conf_dir !RABBITMQ_GENERATED_CONFIG_DIR! ^
+ -conf_script_dir !CONF_SCRIPT_DIR:\=/! ^
+ -conf_schema_dir !RABBITMQ_SCHEMA_DIR!
+ if exist "!RABBITMQ_ADVANCED_CONFIG_FILE!.config" (
+ set RABBITMQ_CONFIG_ARG=!RABBITMQ_CONFIG_ARG! ^
+ -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+ -config "!RABBITMQ_ADVANCED_CONFIG_FILE!"
+ )
) else (
- set RABBITMQ_CONFIG_ARG=
+ if exist "!RABBITMQ_CONFIG_FILE!.config" (
+ set RABBITMQ_CONFIG_ARG=-config "!RABBITMQ_CONFIG_FILE!"
+ ) else if exist "!RABBITMQ_CONFIG_FILE!.conf" (
+ set RABBITMQ_CONFIG_ARG=-conf "!RABBITMQ_CONFIG_FILE!" ^
+ -conf_dir !RABBITMQ_GENERATED_CONFIG_DIR! ^
+ -conf_script_dir !CONF_SCRIPT_DIR:\=/! ^
+ -conf_schema_dir !RABBITMQ_SCHEMA_DIR!
+ if exist "!RABBITMQ_ADVANCED_CONFIG_FILE!.config" (
+ set RABBITMQ_CONFIG_ARG=!RABBITMQ_CONFIG_ARG! ^
+ -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+ -config "!RABBITMQ_ADVANCED_CONFIG_FILE!"
+ )
+ )
)
set RABBITMQ_LISTEN_ARG=
@@ -71,22 +114,15 @@ if not "!RABBITMQ_NODE_IP_ADDRESS!"=="" ( )
)
-REM If $RABBITMQ_LOGS is '-', send all log messages to stdout. Likewise
-REM for RABBITMQ_SASL_LOGS. This is particularily useful for Docker
-REM images.
+REM If $RABBITMQ_LOGS is '-', send all log messages to stdout. This is
+REM particularily useful for Docker images.
if "!RABBITMQ_LOGS!" == "-" (
- set RABBIT_ERROR_LOGGER=tty
-) else (
- set RABBIT_ERROR_LOGGER={file,\""!RABBITMQ_LOGS:\=/!"\"}
-)
-
-if "!RABBITMQ_SASL_LOGS!" == "-" (
set SASL_ERROR_LOGGER=tty
- set RABBIT_SASL_ERROR_LOGGER=tty
+ set RABBIT_LAGER_HANDLER=tty
) else (
set SASL_ERROR_LOGGER=false
- set RABBIT_SASL_ERROR_LOGGER={file,\""!RABBITMQ_SASL_LOGS:\=/!"\"}
+ set RABBIT_LAGER_HANDLER=\""!RABBITMQ_LOGS:\=/!"\"
)
set RABBITMQ_START_RABBIT=
@@ -99,11 +135,11 @@ if "!RABBITMQ_NODE_ONLY!"=="" ( if "!RABBITMQ_IO_THREAD_POOL_SIZE!"=="" (
set RABBITMQ_IO_THREAD_POOL_SIZE=64
-)
+)
set ENV_OK=true
-CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE!
+CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE!
CALL :check_not_empty "RABBITMQ_NAME_TYPE" !RABBITMQ_NAME_TYPE!
CALL :check_not_empty "RABBITMQ_NODENAME" !RABBITMQ_NODENAME!
@@ -126,8 +162,8 @@ if "!ENV_OK!"=="false" ( !RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS! ^
-sasl errlog_type error ^
-sasl sasl_error_logger !SASL_ERROR_LOGGER! ^
--rabbit error_logger !RABBIT_ERROR_LOGGER! ^
--rabbit sasl_error_logger !RABBIT_SASL_ERROR_LOGGER! ^
+-rabbit lager_log_root \""!RABBITMQ_LOG_BASE:\=/!"\" ^
+-rabbit lager_handler !RABBIT_LAGER_HANDLER! ^
-rabbit enabled_plugins_file \""!RABBITMQ_ENABLED_PLUGINS_FILE:\=/!"\" ^
-rabbit plugins_dir \""!RABBITMQ_PLUGINS_DIR:\=/!"\" ^
-rabbit plugins_expand_dir \""!RABBITMQ_PLUGINS_EXPAND_DIR:\=/!"\" ^
@@ -145,10 +181,14 @@ EXIT /B 0 if "%~2"=="" (
ECHO "Error: ENV variable should be defined: %1. Please check rabbitmq-env and rabbitmq-defaults, and !RABBITMQ_CONF_ENV_FILE! script files. Check also your Environment Variables settings"
set ENV_OK=false
- EXIT /B 78
+ EXIT /B 78
)
EXIT /B 0
+:get_noex
+set "%~2=%~dpn1"
+EXIT /B 0
+
endlocal
endlocal
diff --git a/scripts/rabbitmq-service.bat b/scripts/rabbitmq-service.bat index f8a8d5a464..c9e404db46 100644 --- a/scripts/rabbitmq-service.bat +++ b/scripts/rabbitmq-service.bat @@ -21,6 +21,7 @@ rem Preserve values that might contain exclamation marks before rem enabling delayed expansion
set TN0=%~n0
set TDP0=%~dp0
+set CONF_SCRIPT_DIR="%~dp0"
set P1=%1
setlocal enabledelayedexpansion
@@ -105,7 +106,7 @@ if not exist "!RABBITMQ_BASE!" ( )
set ENV_OK=true
-CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE!
+CALL :check_not_empty "RABBITMQ_BOOT_MODULE" !RABBITMQ_BOOT_MODULE!
CALL :check_not_empty "RABBITMQ_NAME_TYPE" !RABBITMQ_NAME_TYPE!
CALL :check_not_empty "RABBITMQ_NODENAME" !RABBITMQ_NODENAME!
@@ -123,10 +124,19 @@ if errorlevel 1 ( set RABBITMQ_EBIN_ROOT=!RABBITMQ_HOME!\ebin
+CALL :get_noex !RABBITMQ_ADVANCED_CONFIG_FILE! RABBITMQ_ADVANCED_CONFIG_FILE_NOEX
+
+if "!RABBITMQ_ADVANCED_CONFIG_FILE!" == "!RABBITMQ_ADVANCED_CONFIG_FILE_NOEX!.config" (
+ set RABBITMQ_ADVANCED_CONFIG_FILE=!RABBITMQ_ADVANCED_CONFIG_FILE_NOEX!
+)
+
"!ERLANG_HOME!\bin\erl.exe" ^
-pa "!RABBITMQ_EBIN_ROOT!" ^
-noinput -hidden ^
-s rabbit_prelaunch ^
+ -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+ -rabbit enabled_plugins_file "!RABBITMQ_ENABLED_PLUGINS_FILE!" ^
+ -rabbit plugins_dir "!$RABBITMQ_PLUGINS_DIR!" ^
!RABBITMQ_NAME_TYPE! rabbitmqprelaunch!RANDOM!!TIME:~9!
if ERRORLEVEL 3 (
@@ -141,18 +151,54 @@ if ERRORLEVEL 3 ( set RABBITMQ_DIST_ARG=-kernel inet_dist_listen_min !RABBITMQ_DIST_PORT! -kernel inet_dist_listen_max !RABBITMQ_DIST_PORT!
)
- REM Try to create config file, if it doesn't exist
+if not exist "!RABBITMQ_SCHEMA_DIR!" (
+ mkdir "!RABBITMQ_SCHEMA_DIR!"
+)
+
+if not exist "!RABBITMQ_SCHEMA_DIR!\rabbitmq.schema" (
+ copy "!RABBITMQ_HOME!\priv\schema\rabbitmq.schema" "!RABBITMQ_SCHEMA_DIR!\rabbitmq.schema"
+)
+ REM Try to create advanced config file, if it doesn't exist
REM It still can fail to be created, but at least not for default install
-if not exist "!RABBITMQ_CONFIG_FILE!.config" (
- echo []. > !RABBITMQ_CONFIG_FILE!.config
+if not exist "!RABBITMQ_ADVANCED_CONFIG_FILE!.config" (
+ echo []. > !RABBITMQ_ADVANCED_CONFIG_FILE!.config
)
-if exist "!RABBITMQ_CONFIG_FILE!.config" (
- set RABBITMQ_CONFIG_ARG=-config "!RABBITMQ_CONFIG_FILE!"
+CALL :get_noex !RABBITMQ_CONFIG_FILE! RABBITMQ_CONFIG_FILE_NOEX
+
+if "!RABBITMQ_CONFIG_FILE!" == "!RABBITMQ_CONFIG_FILE_NOEX!.config" (
+ if exist "!RABBITMQ_CONFIG_FILE!" (
+ set RABBITMQ_CONFIG_ARG=-config "!RABBITMQ_CONFIG_FILE_NOEX!"
+ )
+) else if "!RABBITMQ_CONFIG_FILE!" == "!RABBITMQ_CONFIG_FILE_NOEX!.conf" (
+ set RABBITMQ_CONFIG_ARG=-conf "!RABBITMQ_CONFIG_FILE_NOEX!" ^
+ -conf_dir !RABBITMQ_GENERATED_CONFIG_DIR! ^
+ -conf_script_dir !CONF_SCRIPT_DIR:\=/! ^
+ -conf_schema_dir !RABBITMQ_SCHEMA_DIR!
+ if exist "!RABBITMQ_ADVANCED_CONFIG_FILE!.config" (
+ set RABBITMQ_CONFIG_ARG=!RABBITMQ_CONFIG_ARG! ^
+ -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+ -config "!RABBITMQ_ADVANCED_CONFIG_FILE!"
+ )
) else (
- set RABBITMQ_CONFIG_ARG=
+ if exist "!RABBITMQ_CONFIG_FILE!.config" (
+ set RABBITMQ_CONFIG_ARG=-config "!RABBITMQ_CONFIG_FILE!"
+ ) else (
+ rem Always specify generated config arguments, we cannot
+ rem assume .conf file is available
+ set RABBITMQ_CONFIG_ARG=-conf "!RABBITMQ_CONFIG_FILE!" ^
+ -conf_dir !RABBITMQ_GENERATED_CONFIG_DIR! ^
+ -conf_script_dir !CONF_SCRIPT_DIR:\=/! ^
+ -conf_schema_dir !RABBITMQ_SCHEMA_DIR!
+ if exist "!RABBITMQ_ADVANCED_CONFIG_FILE!.config" (
+ set RABBITMQ_CONFIG_ARG=!RABBITMQ_CONFIG_ARG! ^
+ -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+ -config "!RABBITMQ_ADVANCED_CONFIG_FILE!"
+ )
+ )
)
+
set RABBITMQ_LISTEN_ARG=
if not "!RABBITMQ_NODE_IP_ADDRESS!"=="" (
if not "!RABBITMQ_NODE_PORT!"=="" (
@@ -160,6 +206,12 @@ if not "!RABBITMQ_NODE_IP_ADDRESS!"=="" ( )
)
+if "!RABBITMQ_LOGS!" == "-" (
+ set RABBIT_LAGER_HANDLER=tty
+) else (
+ set RABBIT_LAGER_HANDLER=\""!RABBITMQ_LOGS:\=/!"\"
+)
+
set RABBITMQ_START_RABBIT=
if "!RABBITMQ_NODE_ONLY!"=="" (
set RABBITMQ_START_RABBIT=-s "!RABBITMQ_BOOT_MODULE!" boot
@@ -186,8 +238,8 @@ set ERLANG_SERVICE_ARGUMENTS= ^ !RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS! ^
-sasl errlog_type error ^
-sasl sasl_error_logger false ^
--rabbit error_logger {file,\""!RABBITMQ_LOGS:\=/!"\"} ^
--rabbit sasl_error_logger {file,\""!RABBITMQ_SASL_LOGS:\=/!"\"} ^
+-rabbit lager_log_root \""!RABBITMQ_LOG_BASE:\=/!"\" ^
+-rabbit lager_handler !RABBIT_LAGER_HANDLER! ^
-rabbit enabled_plugins_file \""!RABBITMQ_ENABLED_PLUGINS_FILE:\=/!"\" ^
-rabbit plugins_dir \""!RABBITMQ_PLUGINS_DIR:\=/!"\" ^
-rabbit plugins_expand_dir \""!RABBITMQ_PLUGINS_EXPAND_DIR:\=/!"\" ^
@@ -234,9 +286,13 @@ EXIT /B 0 if "%~2"=="" (
ECHO "Error: ENV variable should be defined: %1. Please check rabbitmq-env, rabbitmq-default, and !RABBITMQ_CONF_ENV_FILE! script files. Check also your Environment Variables settings"
set ENV_OK=false
- EXIT /B 78
+ EXIT /B 78
)
EXIT /B 0
+:get_noex
+set "%~2=%~dpn1"
+EXIT /B 0
+
endlocal
endlocal
diff --git a/src/file_handle_cache.erl b/src/file_handle_cache.erl index e4af1e8c1a..3f5232a657 100644 --- a/src/file_handle_cache.erl +++ b/src/file_handle_cache.erl @@ -540,12 +540,12 @@ clear(Ref) -> end). set_maximum_since_use(MaximumAge) -> - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), case lists:foldl( fun ({{Ref, fhc_handle}, Handle = #handle { hdl = Hdl, last_used_at = Then }}, Rep) -> case Hdl =/= closed andalso - time_compat:convert_time_unit(Now - Then, + erlang:convert_time_unit(Now - Then, native, micro_seconds) >= MaximumAge of @@ -722,7 +722,7 @@ get_or_reopen(RefNewOrReopens) -> {ok, [Handle || {_Ref, Handle} <- OpenHdls]}; {OpenHdls, ClosedHdls} -> Oldest = oldest(get_age_tree(), - fun () -> time_compat:monotonic_time() end), + fun () -> erlang:monotonic_time() end), case gen_server2:call(?SERVER, {open, self(), length(ClosedHdls), Oldest}, infinity) of ok -> @@ -758,7 +758,7 @@ reopen([{Ref, NewOrReopen, Handle = #handle { hdl = closed, end, case prim_file:open(Path, Mode) of {ok, Hdl} -> - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), {{ok, _Offset}, Handle1} = maybe_seek(Offset, reset_read_buffer( Handle#handle{hdl = Hdl, @@ -794,7 +794,7 @@ sort_handles([{Ref, _} | RefHdls], RefHdlsA, [{Ref, Handle} | RefHdlsB], Acc) -> sort_handles(RefHdls, RefHdlsA, RefHdlsB, [Handle | Acc]). put_handle(Ref, Handle = #handle { last_used_at = Then }) -> - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), age_tree_update(Then, Now, Ref), put({Ref, fhc_handle}, Handle #handle { last_used_at = Now }). @@ -1435,14 +1435,14 @@ reduce(State = #fhc_state { open_pending = OpenPending, elders = Elders, clients = Clients, timer_ref = TRef }) -> - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), {CStates, Sum, ClientCount} = ets:foldl(fun ({Pid, Eldest}, {CStatesAcc, SumAcc, CountAcc} = Accs) -> [#cstate { pending_closes = PendingCloses, opened = Opened, blocked = Blocked } = CState] = ets:lookup(Clients, Pid), - TimeDiff = time_compat:convert_time_unit( + TimeDiff = erlang:convert_time_unit( Now - Eldest, native, micro_seconds), case Blocked orelse PendingCloses =:= Opened of true -> Accs; @@ -1481,7 +1481,7 @@ notify_age(CStates, AverageAge) -> notify_age0(Clients, CStates, Required) -> case [CState || CState <- CStates, CState#cstate.callback =/= undefined] of [] -> ok; - Notifications -> S = rand_compat:uniform(length(Notifications)), + Notifications -> S = rand:uniform(length(Notifications)), {L1, L2} = lists:split(S, Notifications), notify(Clients, Required, L2 ++ L1) end. diff --git a/src/file_handle_cache_stats.erl b/src/file_handle_cache_stats.erl index 12a78f805e..f639881a6f 100644 --- a/src/file_handle_cache_stats.erl +++ b/src/file_handle_cache_stats.erl @@ -59,8 +59,8 @@ get() -> lists:sort(ets:tab2list(?TABLE)). timer_tc(Thunk) -> - T1 = time_compat:monotonic_time(), + T1 = erlang:monotonic_time(), Res = Thunk(), - T2 = time_compat:monotonic_time(), - Diff = time_compat:convert_time_unit(T2 - T1, native, micro_seconds), + T2 = erlang:monotonic_time(), + Diff = erlang:convert_time_unit(T2 - T1, native, micro_seconds), {Diff, Res}. diff --git a/src/gm.erl b/src/gm.erl index 41aa01f04d..3a1459fea4 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -1076,7 +1076,7 @@ join_group(Self, GroupName, #gm_group { members = Members } = Group, TxnFun) -> prune_or_create_group(Self, GroupName, TxnFun), TxnFun); Alive -> - Left = lists:nth(rand_compat:uniform(length(Alive)), Alive), + Left = lists:nth(rand:uniform(length(Alive)), Alive), Handler = fun () -> join_group( diff --git a/src/lqueue.erl b/src/lqueue.erl index fc7157dff1..0652061075 100644 --- a/src/lqueue.erl +++ b/src/lqueue.erl @@ -27,9 +27,7 @@ -export_type([?MODULE/0]). --include_lib("rabbit_common/include/old_builtin_types.hrl"). - --opaque ?MODULE() :: {non_neg_integer(), ?QUEUE_TYPE()}. +-opaque ?MODULE() :: {non_neg_integer(), queue:queue()}. -type value() :: any(). -type result() :: 'empty' | {'value', value()}. diff --git a/src/rabbit.app.src b/src/rabbit.app.src index dd38ad4072..c06f7630fa 100644 --- a/src/rabbit.app.src +++ b/src/rabbit.app.src @@ -1,4 +1,5 @@ -{application, rabbit, %% -*- erlang -*- +%% -*- erlang -*- +{application, rabbit, [{description, "RabbitMQ"}, {id, "RabbitMQ"}, {vsn, "0.0.0"}, @@ -9,7 +10,9 @@ rabbit_router, rabbit_sup, rabbit_direct_client_sup]}, - {applications, [kernel, stdlib, sasl, mnesia, rabbit_common, ranch, os_mon, xmerl]}, + %% FIXME: Remove goldrush, once rabbit_plugins.erl knows how to ignore + %% indirect dependencies of rabbit. + {applications, [kernel, stdlib, sasl, mnesia, goldrush, lager, rabbit_common, ranch, os_mon, xmerl]}, %% we also depend on crypto, public_key and ssl but they shouldn't be %% in here as we don't actually want to start it {mod, {rabbit, []}}, @@ -41,7 +44,6 @@ {default_permissions, [<<".*">>, <<".*">>, <<".*">>]}, {loopback_users, [<<"guest">>]}, {password_hashing_module, rabbit_password_hashing_sha256}, - {cluster_nodes, {[], disc}}, {server_properties, []}, {collect_statistics, none}, {collect_statistics_interval, 5000}, @@ -100,12 +102,19 @@ %% see rabbitmq-server#248 %% and rabbitmq-server#667 {channel_operation_timeout, 15000}, + + %% see rabbitmq-server#486 + {peer_discovery_backend, rabbit_peer_discovery_classic_config}, + %% used by rabbit_peer_discovery_classic_config + {cluster_nodes, {[], disc}}, + {config_entry_decoder, [ {cipher, aes_cbc256}, {hash, sha512}, {iterations, 1000}, {passphrase, undefined} ]}, + %% rabbitmq-server-973 {lazy_queue_explicit_gc_run_operation_threshold, 250} ]}]}. diff --git a/src/rabbit.erl b/src/rabbit.erl index 1f0df1ad06..f10eb463ab 100644 --- a/src/rabbit.erl +++ b/src/rabbit.erl @@ -20,11 +20,17 @@ -export([start/0, boot/0, stop/0, stop_and_halt/0, await_startup/0, status/0, is_running/0, - is_running/1, environment/0, rotate_logs/1, force_event_refresh/1, + is_running/1, environment/0, rotate_logs/0, force_event_refresh/1, start_fhc/0]). -export([start/2, stop/1, prep_stop/1]). -export([start_apps/1, stop_apps/1]). --export([log_location/1, config_files/0, decrypt_config/2]). %% for testing and mgmt-agent +-export([log_locations/0, config_files/0, decrypt_config/2]). %% for testing and mgmt-agent + +-ifdef(TEST). + +-export([start_logger/0]). + +-endif. %%--------------------------------------------------------------------------- %% Boot steps. @@ -172,6 +178,11 @@ {mfa, {rabbit_direct, boot, []}}, {requires, log_relay}]}). +-rabbit_boot_step({connection_tracking, + [{description, "sets up internal storage for node-local connections"}, + {mfa, {rabbit_connection_tracking, boot, []}}, + {requires, log_relay}]}). + -rabbit_boot_step({networking, [{mfa, {rabbit_networking, boot, []}}, {requires, log_relay}]}). @@ -198,9 +209,8 @@ %%---------------------------------------------------------------------------- --type file_suffix() :: binary(). %% this really should be an abstract type --type log_location() :: 'tty' | 'undefined' | file:filename(). +-type log_location() :: string(). -type param() :: atom(). -type app_name() :: atom(). @@ -218,10 +228,10 @@ -spec is_running() -> boolean(). -spec is_running(node()) -> boolean(). -spec environment() -> [{param(), term()}]. --spec rotate_logs(file_suffix()) -> rabbit_types:ok_or_error(any()). +-spec rotate_logs() -> rabbit_types:ok_or_error(any()). -spec force_event_refresh(reference()) -> 'ok'. --spec log_location('sasl' | 'kernel') -> log_location(). +-spec log_locations() -> [log_location()]. -spec start('normal',[]) -> {'error', @@ -254,7 +264,7 @@ start() -> %% restarting the app. ok = ensure_application_loaded(), HipeResult = rabbit_hipe:maybe_hipe_compile(), - ok = ensure_working_log_handlers(), + ok = start_logger(), rabbit_hipe:log_hipe_result(HipeResult), rabbit_node_monitor:prepare_cluster_status_files(), rabbit_mnesia:check_cluster_consistency(), @@ -263,9 +273,10 @@ start() -> boot() -> start_it(fun() -> + ensure_config(), ok = ensure_application_loaded(), HipeResult = rabbit_hipe:maybe_hipe_compile(), - ok = ensure_working_log_handlers(), + ok = start_logger(), rabbit_hipe:log_hipe_result(HipeResult), rabbit_node_monitor:prepare_cluster_status_files(), ok = rabbit_upgrade:maybe_upgrade_mnesia(), @@ -276,6 +287,20 @@ boot() -> broker_start() end). +ensure_config() -> + case rabbit_config:prepare_and_use_config() of + {error, Reason} -> + {Format, Arg} = case Reason of + {generation_error, Error} -> {"~s", [Error]}; + Other -> {"~p", [Other]} + end, + log_boot_error_and_exit(generate_config_file, + "~nConfig file generation failed "++Format, + Arg); + ok -> ok + end. + + broker_start() -> Plugins = rabbit_plugins:setup(), ToBeLoaded = Plugins ++ ?APPS, @@ -403,10 +428,8 @@ start_it(StartFun) -> false -> StartFun() end catch - throw:{could_not_start, _App, _Reason} = Err -> - boot_error(Err, not_available); - _:Reason -> - boot_error(Reason, erlang:get_stacktrace()) + Class:Reason -> + boot_error(Class, Reason) after unlink(Marker), Marker ! stop, @@ -655,27 +678,33 @@ environment(App) -> lists:keysort(1, [P || P = {K, _} <- application:get_all_env(App), not lists:member(K, Ignore)]). -rotate_logs_info("") -> - rabbit_log:info("Reopening logs", []); -rotate_logs_info(Suffix) -> - rabbit_log:info("Rotating logs with suffix '~s'~n", [Suffix]). - -rotate_logs(BinarySuffix) -> - Suffix = binary_to_list(BinarySuffix), - rotate_logs_info(Suffix), - log_rotation_result(rotate_logs(log_location(kernel), - Suffix, - rabbit_error_logger_file_h), - rotate_logs(log_location(sasl), - Suffix, - rabbit_sasl_report_file_h)). +rotate_logs() -> + rabbit_lager:fold_sinks( + fun + (_, [], Acc) -> + Acc; + (SinkName, FileNames, Acc) -> + lager:log(SinkName, info, self(), + "Log file rotation forced", []), + %% FIXME: We use an internal message, understood by + %% lager_file_backend. We should use a proper API, when + %% it's added to Lager. + %% + %% FIXME: This message is asynchronous, therefore this + %% entire call is asynchronous: at the end of this + %% function, we can't guaranty the rotation is completed. + [SinkName ! {rotate, FileName} || FileName <- FileNames], + lager:log(SinkName, info, self(), + "Log file re-opened after forced rotation", []), + Acc + end, ok). %%-------------------------------------------------------------------- start(normal, []) -> case erts_version_check() of ok -> - rabbit_log:info("Starting RabbitMQ ~s on Erlang ~s~n~s~n~s~n", + rabbit_log:info("~n Starting RabbitMQ ~s on Erlang ~s~n ~s~n ~s~n", [rabbit_misc:version(), rabbit_misc:otp_release(), ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE]), {ok, SupPid} = rabbit_sup:start_link(), @@ -700,10 +729,9 @@ prep_stop(_State) -> stop(_) -> ok. --spec boot_error(term(), not_available | [tuple()]) -> no_return(). +-spec boot_error(atom(), term()) -> no_return(). -boot_error({could_not_start, rabbit, {{timeout_waiting_for_tables, _}, _}}, - _Stacktrace) -> +boot_error(_, {could_not_start, rabbit, {{timeout_waiting_for_tables, _}, _}}) -> AllNodes = rabbit_mnesia:cluster_nodes(all), Suffix = "~nBACKGROUND~n==========~n~n" "This cluster node was shut down while other nodes were still running.~n" @@ -722,25 +750,25 @@ boot_error({could_not_start, rabbit, {{timeout_waiting_for_tables, _}, _}}, end, log_boot_error_and_exit( timeout_waiting_for_tables, - Err ++ rabbit_nodes:diagnostics(Nodes) ++ "~n~n", []); -boot_error(Reason, Stacktrace) -> - Fmt = "Error description:~n ~p~n~n" - "Log files (may contain more information):~n ~s~n ~s~n~n", - Args = [Reason, log_location(kernel), log_location(sasl)], - boot_error(Reason, Fmt, Args, Stacktrace). - --spec boot_error(term(), string(), [any()], not_available | [tuple()]) -> - no_return(). - -boot_error(Reason, Fmt, Args, not_available) -> - log_boot_error_and_exit(Reason, Fmt, Args); -boot_error(Reason, Fmt, Args, Stacktrace) -> - log_boot_error_and_exit(Reason, Fmt ++ "Stack trace:~n ~p~n~n", - Args ++ [Stacktrace]). + "~n" ++ Err ++ rabbit_nodes:diagnostics(Nodes), []); +boot_error(Class, {error, {cannot_log_to_file, _, _}} = Reason) -> + log_boot_error_and_exit( + Reason, + "~nError description:~s", + [lager:pr_stacktrace(erlang:get_stacktrace(), {Class, Reason})]); +boot_error(Class, Reason) -> + LogLocations = log_locations(), + log_boot_error_and_exit( + Reason, + "~nError description:~s" + "~nLog file(s) (may contain more information):~n" ++ + lists:flatten([" ~s~n" || _ <- lists:seq(1, length(LogLocations))]), + [lager:pr_stacktrace(erlang:get_stacktrace(), {Class, Reason})] ++ + LogLocations). log_boot_error_and_exit(Reason, Format, Args) -> - io:format("~n~nBOOT FAILED~n===========~n~n" ++ Format, Args), - rabbit_log:info(Format, Args), + rabbit_log:error(Format, Args), + io:format("~nBOOT FAILED~n===========~n" ++ Format ++ "~n", Args), timer:sleep(1000), exit(Reason). @@ -784,75 +812,12 @@ insert_default_data() -> %%--------------------------------------------------------------------------- %% logging -ensure_working_log_handlers() -> - Handlers = gen_event:which_handlers(error_logger), - ok = ensure_working_log_handler(error_logger_tty_h, - rabbit_error_logger_file_h, - error_logger_tty_h, - log_location(kernel), - Handlers), - - ok = ensure_working_log_handler(sasl_report_tty_h, - rabbit_sasl_report_file_h, - sasl_report_tty_h, - log_location(sasl), - Handlers), +start_logger() -> + rabbit_lager:start_logger(), ok. -ensure_working_log_handler(OldHandler, NewHandler, TTYHandler, - LogLocation, Handlers) -> - case LogLocation of - undefined -> ok; - tty -> case lists:member(TTYHandler, Handlers) of - true -> ok; - false -> - throw({error, {cannot_log_to_tty, - TTYHandler, not_installed}}) - end; - _ -> case lists:member(NewHandler, Handlers) of - true -> ok; - false -> case rotate_logs(LogLocation, "", - OldHandler, NewHandler) of - ok -> ok; - {error, Reason} -> - throw({error, {cannot_log_to_file, - LogLocation, Reason}}) - end - end - end. - -log_location(Type) -> - case application:get_env(rabbit, case Type of - kernel -> error_logger; - sasl -> sasl_error_logger - end) of - {ok, {file, File}} -> File; - {ok, false} -> undefined; - {ok, tty} -> tty; - {ok, silent} -> undefined; - {ok, Bad} -> throw({error, {cannot_log_to_file, Bad}}); - _ -> undefined - end. - -rotate_logs(File, Suffix, Handler) -> - rotate_logs(File, Suffix, Handler, Handler). - -rotate_logs(undefined, _Suffix, _OldHandler, _NewHandler) -> ok; -rotate_logs(tty, _Suffix, _OldHandler, _NewHandler) -> ok; -rotate_logs(File, Suffix, OldHandler, NewHandler) -> - gen_event:swap_handler(error_logger, - {OldHandler, swap}, - {NewHandler, {File, Suffix}}). - -log_rotation_result({error, MainLogError}, {error, SaslLogError}) -> - {error, {{cannot_rotate_main_logs, MainLogError}, - {cannot_rotate_sasl_logs, SaslLogError}}}; -log_rotation_result({error, MainLogError}, ok) -> - {error, {cannot_rotate_main_logs, MainLogError}}; -log_rotation_result(ok, {error, SaslLogError}) -> - {error, {cannot_rotate_sasl_logs, SaslLogError}}; -log_rotation_result(ok, ok) -> - ok. +log_locations() -> + rabbit_lager:log_locations(). force_event_refresh(Ref) -> rabbit_direct:force_event_refresh(Ref), @@ -864,19 +829,17 @@ force_event_refresh(Ref) -> %% misc log_broker_started(Plugins) -> - rabbit_log:with_local_io( - fun() -> - PluginList = iolist_to_binary([rabbit_misc:format(" * ~s~n", [P]) - || P <- Plugins]), - rabbit_log:info( - "Server startup complete; ~b plugins started.~n~s", - [length(Plugins), PluginList]), - io:format(" completed with ~p plugins.~n", [length(Plugins)]) - end). + PluginList = iolist_to_binary([rabbit_misc:format(" * ~s~n", [P]) + || P <- Plugins]), + Message = string:strip(rabbit_misc:format( + "Server startup complete; ~b plugins started.~n~s", + [length(Plugins), PluginList]), right, $\n), + rabbit_log:info(Message), + io:format(" completed with ~p plugins.~n", [length(Plugins)]). erts_version_check() -> ERTSVer = erlang:system_info(version), - OTPRel = erlang:system_info(otp_release), + OTPRel = rabbit_misc:otp_release(), case rabbit_misc:version_compare(?ERTS_MINIMUM, ERTSVer, lte) of true when ?ERTS_MINIMUM =/= ERTSVer -> ok; @@ -897,31 +860,45 @@ erts_version_check() -> print_banner() -> {ok, Product} = application:get_key(id), {ok, Version} = application:get_key(vsn), - io:format("~n ~s ~s. ~s" - "~n ## ## ~s" - "~n ## ##" - "~n ########## Logs: ~s" - "~n ###### ## ~s" - "~n ##########" - "~n Starting broker..." + {LogFmt, LogLocations} = case log_locations() of + [_ | Tail] = LL -> + LF = lists:flatten(["~n ~s" + || _ <- lists:seq(1, length(Tail))]), + {LF, LL}; + [] -> + {"", ["(none)"]} + end, + io:format("~n ## ##" + "~n ## ## ~s ~s. ~s" + "~n ########## ~s" + "~n ###### ##" + "~n ########## Logs: ~s" ++ + LogFmt ++ + "~n~n Starting broker..." "~n", - [Product, Version, ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE, - log_location(kernel), log_location(sasl)]). + [Product, Version, ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE] ++ + LogLocations). log_banner() -> + {FirstLog, OtherLogs} = case log_locations() of + [Head | Tail] -> + {Head, [{"", F} || F <- Tail]}; + [] -> + {"(none)", []} + end, Settings = [{"node", node()}, {"home dir", home_dir()}, {"config file(s)", config_files()}, {"cookie hash", rabbit_nodes:cookie_hash()}, - {"log", log_location(kernel)}, - {"sasl log", log_location(sasl)}, - {"database dir", rabbit_mnesia:dir()}], + {"log(s)", FirstLog}] ++ + OtherLogs ++ + [{"database dir", rabbit_mnesia:dir()}], DescrLen = 1 + lists:max([length(K) || {K, _V} <- Settings]), Format = fun (K, V) -> rabbit_misc:format( - "~-" ++ integer_to_list(DescrLen) ++ "s: ~s~n", [K, V]) + " ~-" ++ integer_to_list(DescrLen) ++ "s: ~s~n", [K, V]) end, - Banner = iolist_to_binary( + Banner = string:strip(lists:flatten( [case S of {"config file(s)" = K, []} -> Format(K, "(none)"); @@ -929,8 +906,8 @@ log_banner() -> [Format(K, V0) | [Format("", V) || V <- Vs]]; {K, V} -> Format(K, V) - end || S <- Settings]), - rabbit_log:info("~s", [Banner]). + end || S <- Settings]), right, $\n), + rabbit_log:info("~n~s", [Banner]). warn_if_kernel_config_dubious() -> case os:type() of @@ -1048,32 +1025,7 @@ home_dir() -> end. config_files() -> - Abs = fun (F) -> - filename:absname(filename:rootname(F, ".config") ++ ".config") - end, - case init:get_argument(config) of - {ok, Files} -> [Abs(File) || [File] <- Files]; - error -> case config_setting() of - none -> []; - File -> [Abs(File) ++ " (not found)"] - end - end. - -%% This is a pain. We want to know where the config file is. But we -%% can't specify it on the command line if it is missing or the VM -%% will fail to start, so we need to find it by some mechanism other -%% than init:get_arguments/0. We can look at the environment variable -%% which is responsible for setting it... but that doesn't work for a -%% Windows service since the variable can change and the service not -%% be reinstalled, so in that case we add a magic application env. -config_setting() -> - case application:get_env(rabbit, windows_service_config) of - {ok, File1} -> File1; - undefined -> case os:getenv("RABBITMQ_CONFIG_FILE") of - false -> none; - File2 -> File2 - end - end. + rabbit_config:config_files(). %% We don't want this in fhc since it references rabbit stuff. And we can't put %% this in the bootstep directly. @@ -1099,9 +1051,8 @@ ensure_working_fhc() -> {ok, true} -> "ON"; {ok, false} -> "OFF" end, - rabbit_log:info( - "FHC read buffering: ~s~n" - "FHC write buffering: ~s~n", [ReadBuf, WriteBuf]), + rabbit_log:info("FHC read buffering: ~s~n", [ReadBuf]), + rabbit_log:info("FHC write buffering: ~s~n", [WriteBuf]), Filename = filename:join(code:lib_dir(kernel, ebin), "kernel.app"), {ok, Fd} = file_handle_cache:open(Filename, [raw, binary, read], []), {ok, _} = file_handle_cache:read(Fd, 1), diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl index bfa868c651..25555156d6 100644 --- a/src/rabbit_amqqueue_process.erl +++ b/src/rabbit_amqqueue_process.erl @@ -85,6 +85,10 @@ %% e.g. message expiration messages from previously set up timers %% that may or may not be still valid args_policy_version, + %% used to discard outdated/superseded policy updates, + %% e.g. when policies are applied concurrently. See + %% https://github.com/rabbitmq/rabbitmq-server/issues/803 for one + %% example. mirroring_policy_version = 0, %% running | flow | idle status @@ -95,7 +99,7 @@ -spec info_keys() -> rabbit_types:info_keys(). -spec init_with_backing_queue_state (rabbit_types:amqqueue(), atom(), tuple(), any(), - [rabbit_types:delivery()], pmon:pmon(), ?DICT_TYPE()) -> + [rabbit_types:delivery()], pmon:pmon(), dict:dict()) -> #q{}. %%---------------------------------------------------------------------------- @@ -103,6 +107,8 @@ -define(STATISTICS_KEYS, [name, policy, + operator_policy, + effective_policy_definition, exclusive_consumer_pid, exclusive_consumer_tag, messages_ready, @@ -174,6 +180,7 @@ init_it(Recover, From, State = #q{q = #amqqueue{exclusive_owner = Owner}}) -> {_, Terms} = recovery_status(Recover), BQS = bq_init(BQ, Q, Terms), %% Rely on terminate to delete the queue. + log_delete_exclusive(Owner, State), {stop, {shutdown, missing_owner}, State#q{backing_queue = BQ, backing_queue_state = BQS}} end. @@ -462,7 +469,7 @@ ensure_ttl_timer(undefined, State) -> State; ensure_ttl_timer(Expiry, State = #q{ttl_timer_ref = undefined, args_policy_version = Version}) -> - After = (case Expiry - time_compat:os_system_time(micro_seconds) of + After = (case Expiry - os:system_time(micro_seconds) of V when V > 0 -> V + 999; %% always fire later _ -> 0 end) div 1000, @@ -733,7 +740,13 @@ handle_ch_down(DownPid, State = #q{consumers = Consumers, exclusive_consumer = Holder1}, notify_decorators(State2), case should_auto_delete(State2) of - true -> {stop, State2}; + true -> + log_auto_delete( + io_lib:format( + "because all of its consumers (~p) were on a channel that was closed", + [length(ChCTags)]), + State), + {stop, State2}; false -> {ok, requeue_and_run(ChAckTags, ensure_expiry_timer(State2))} end @@ -782,7 +795,7 @@ calculate_msg_expiry(#basic_message{content = Content}, TTL) -> {ok, MsgTTL} = rabbit_basic:parse_expiration(Props), case lists:min([TTL, MsgTTL]) of undefined -> undefined; - T -> time_compat:os_system_time(micro_seconds) + T * 1000 + T -> os:system_time(micro_seconds) + T * 1000 end. %% Logically this function should invoke maybe_send_drained/2. @@ -793,7 +806,7 @@ calculate_msg_expiry(#basic_message{content = Content}, TTL) -> drop_expired_msgs(State) -> case is_empty(State) of true -> State; - false -> drop_expired_msgs(time_compat:os_system_time(micro_seconds), + false -> drop_expired_msgs(os:system_time(micro_seconds), State) end. @@ -876,6 +889,16 @@ i(policy, #q{q = Q}) -> none -> ''; Policy -> Policy end; +i(operator_policy, #q{q = Q}) -> + case rabbit_policy:name_op(Q) of + none -> ''; + Policy -> Policy + end; +i(effective_policy_definition, #q{q = Q}) -> + case rabbit_policy:effective_definition(Q) of + undefined -> []; + Def -> Def + end; i(exclusive_consumer_pid, #q{exclusive_consumer = none}) -> ''; i(exclusive_consumer_pid, #q{exclusive_consumer = {ChPid, _ConsumerTag}}) -> @@ -976,6 +999,7 @@ prioritise_call(Msg, _From, _Len, State) -> prioritise_cast(Msg, _Len, State) -> case Msg of delete_immediately -> 8; + {delete_exclusive, _Pid} -> 8; {set_ram_duration_target, _Duration} -> 8; {set_maximum_since_use, _Age} -> 8; {run_backing_queue, _Mod, _Fun} -> 6; @@ -1110,7 +1134,13 @@ handle_call({basic_cancel, ChPid, ConsumerTag, OkMsg}, _From, notify_decorators(State1), case should_auto_delete(State1) of false -> reply(ok, ensure_expiry_timer(State1)); - true -> stop(ok, State1) + true -> + log_auto_delete( + io_lib:format( + "because its last consumer with tag '~s' was cancelled", + [ConsumerTag]), + State), + stop(ok, State1) end end; @@ -1222,6 +1252,10 @@ handle_cast({reject, false, AckTags, ChPid}, State) -> end) end, fun () -> ack(AckTags, ChPid, State) end)); +handle_cast({delete_exclusive, ConnPid}, State) -> + log_delete_exclusive(ConnPid, State), + stop(State); + handle_cast(delete_immediately, State) -> stop(State); @@ -1340,6 +1374,7 @@ handle_info({'DOWN', _MonitorRef, process, DownPid, _Reason}, %% match what people expect (see bug 21824). However we need this %% monitor-and-async- delete in case the connection goes away %% unexpectedly. + log_delete_exclusive(DownPid, State), stop(State); handle_info({'DOWN', _MonitorRef, process, DownPid, _Reason}, State) -> @@ -1395,7 +1430,7 @@ handle_pre_hibernate(State = #q{backing_queue = BQ, State, #q.stats_timer, fun () -> emit_stats(State, [{idle_since, - time_compat:os_system_time(milli_seconds)}, + os:system_time(milli_seconds)}, {consumer_utilisation, ''}]) end), State1 = rabbit_event:stop_stats_timer(State#q{backing_queue_state = BQS3}, @@ -1404,6 +1439,20 @@ handle_pre_hibernate(State = #q{backing_queue = BQ, format_message_queue(Opt, MQ) -> rabbit_misc:format_message_queue(Opt, MQ). +log_delete_exclusive({ConPid, _ConRef}, State) -> + log_delete_exclusive(ConPid, State); +log_delete_exclusive(ConPid, #q{ q = #amqqueue{ name = Resource } }) -> + #resource{ name = QName, virtual_host = VHost } = Resource, + rabbit_log_queue:debug("Deleting exclusive queue '~s' in vhost '~s' " ++ + "because its declaring connection ~p was closed", + [QName, VHost, ConPid]). + +log_auto_delete(Reason, #q{ q = #amqqueue{ name = Resource } }) -> + #resource{ name = QName, virtual_host = VHost } = Resource, + rabbit_log_queue:debug("Deleting auto-delete queue '~s' in vhost '~s' " ++ + Reason, + [QName, VHost]). + needs_update_mirroring(Q, Version) -> {ok, UpQ} = rabbit_amqqueue:lookup(Q#amqqueue.name), DBVersion = UpQ#amqqueue.policy_version, @@ -1412,6 +1461,7 @@ needs_update_mirroring(Q, Version) -> false -> false end. + update_mirroring(Policy, State = #q{backing_queue = BQ}) -> case update_to(Policy, BQ) of start_mirroring -> @@ -1454,3 +1504,7 @@ update_ha_mode(State) -> {ok, Q} = rabbit_amqqueue:lookup(qname(State)), ok = rabbit_mirror_queue_misc:update_mirrors(Q), State. + + + + diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl index 51bc883976..7cebd194a6 100644 --- a/src/rabbit_binding.erl +++ b/src/rabbit_binding.erl @@ -52,7 +52,7 @@ %% TODO this should really be opaque but that seems to confuse 17.1's %% dialyzer into objecting to everything that uses it. --type deletions() :: ?DICT_TYPE(). +-type deletions() :: dict:dict(). -spec recover([rabbit_exchange:name()], [rabbit_amqqueue:name()]) -> 'ok'. diff --git a/src/rabbit_cli.erl b/src/rabbit_cli.erl index c0e5c93247..65e8563ddf 100644 --- a/src/rabbit_cli.erl +++ b/src/rabbit_cli.erl @@ -171,7 +171,7 @@ name_type() -> end. generate_cli_node_name() -> - Base = rabbit_misc:format("rabbitmq-cli-~2..0b", [rand_compat:uniform(100)]), + Base = rabbit_misc:format("rabbitmq-cli-~2..0b", [rand:uniform(100)]), NameAsList = case {name_type(), inet_db:res_option(domain)} of {longnames, []} -> diff --git a/src/rabbit_config.erl b/src/rabbit_config.erl new file mode 100644 index 0000000000..67e7523ec0 --- /dev/null +++ b/src/rabbit_config.erl @@ -0,0 +1,179 @@ +-module(rabbit_config). + +-export([ + generate_config_file/5, + prepare_and_use_config/0, + prepare_config/1, + update_app_config/1, + schema_dir/0, + config_files/0, + get_advanced_config/0 + ]). + +prepare_and_use_config() -> + case legacy_erlang_term_config_used() of + true -> + %% Use .config file + ok; + false -> + case prepare_config(get_confs()) of + ok -> + %% No .conf to generate from + ok; + {ok, GeneratedConfigFile} -> + %% Generated config file + update_app_config(GeneratedConfigFile); + {error, Err} -> + {error, Err} + end + end. + +%% we support both the classic Erlang term +%% config file (rabbitmq.config) as well as rabbitmq.conf +legacy_erlang_term_config_used() -> + case init:get_argument(config) of + error -> false; + {ok, [Config | _]} -> + ConfigFile = Config ++ ".config", + rabbit_file:is_file(ConfigFile) + andalso + get_advanced_config() == none + end. + +get_confs() -> + case init:get_argument(conf) of + {ok, Configs} -> Configs; + _ -> [] + end. + +prepare_config(Configs) -> + case {init:get_argument(conf_dir), init:get_argument(conf_script_dir)} of + {{ok, ConfDir}, {ok, ScriptDir}} -> + ConfFiles = [Config ++ ".conf" || [Config] <- Configs, + rabbit_file:is_file(Config ++ + ".conf")], + case ConfFiles of + [] -> ok; + _ -> + case generate_config_file(ConfFiles, ConfDir, ScriptDir) of + {ok, GeneratedConfigFile} -> + {ok, GeneratedConfigFile}; + {error, Reason} -> + {error, Reason} + end + end; + _ -> ok + end. + +update_app_config(ConfigFile) -> + ok = application_controller:change_application_data([], [ConfigFile]). + +generate_config_file(ConfFiles, ConfDir, ScriptDir) -> + generate_config_file(ConfFiles, ConfDir, ScriptDir, + schema_dir(), get_advanced_config()). + + +generate_config_file(ConfFiles, ConfDir, ScriptDir, SchemaDir, Advanced) -> + prepare_plugin_schemas(SchemaDir), + % SchemaFile = filename:join([ScriptDir, "rabbitmq.schema"]), + Cuttlefish = filename:join([ScriptDir, "cuttlefish"]), + GeneratedDir = filename:join([ConfDir, "generated"]), + + AdvancedConfigArg = case check_advanced_config(Advanced) of + {ok, FileName} -> [" -a ", FileName]; + none -> [] + end, + rabbit_file:recursive_delete([GeneratedDir]), + Command = lists:concat(["escript ", "\"", Cuttlefish, "\"", + " -f rabbitmq -s ", "\"", SchemaDir, "\"", + " -e ", "\"", ConfDir, "\"", + [[" -c ", ConfFile] || ConfFile <- ConfFiles], + AdvancedConfigArg]), + Result = rabbit_misc:os_cmd(Command), + case string:str(Result, " -config ") of + 0 -> {error, {generation_error, Result}}; + _ -> + [OutFile] = rabbit_file:wildcard("rabbitmq.*.config", GeneratedDir), + ResultFile = filename:join([GeneratedDir, "rabbitmq.config"]), + rabbit_file:rename(filename:join([GeneratedDir, OutFile]), + ResultFile), + {ok, ResultFile} + end. + +schema_dir() -> + case init:get_argument(conf_schema_dir) of + {ok, SchemaDir} -> SchemaDir; + _ -> + case code:priv_dir(rabbit) of + {error, bad_name} -> filename:join([".", "priv", "schema"]); + PrivDir -> filename:join([PrivDir, "schema"]) + end + end. + +check_advanced_config(none) -> none; +check_advanced_config(ConfigName) -> + case rabbit_file:is_file(ConfigName) of + true -> {ok, ConfigName}; + false -> none + end. + +get_advanced_config() -> + case init:get_argument(conf_advanced) of + %% There can be only one advanced.config + {ok, [FileName | _]} -> + ConfigName = FileName ++ ".config", + case rabbit_file:is_file(ConfigName) of + true -> ConfigName; + false -> none + end; + _ -> none + end. + + +prepare_plugin_schemas(SchemaDir) -> + case rabbit_file:is_dir(SchemaDir) of + true -> rabbit_plugins:extract_schemas(SchemaDir); + false -> ok + end. + + +config_files() -> + Abs = fun (F, Ex) -> filename:absname(filename:rootname(F, Ex) ++ Ex) end, + case legacy_erlang_term_config_used() of + true -> + case init:get_argument(config) of + {ok, Files} -> [Abs(File, ".config") || [File] <- Files]; + error -> case config_setting() of + none -> []; + File -> [Abs(File, ".config") + ++ + " (not found)"] + end + end; + false -> + ConfFiles = [Abs(File, ".conf") || File <- get_confs()], + AdvancedFiles = case get_advanced_config() of + none -> []; + FileName -> [Abs(FileName, ".config")] + end, + AdvancedFiles ++ ConfFiles + + end. + + +%% This is a pain. We want to know where the config file is. But we +%% can't specify it on the command line if it is missing or the VM +%% will fail to start, so we need to find it by some mechanism other +%% than init:get_arguments/0. We can look at the environment variable +%% which is responsible for setting it... but that doesn't work for a +%% Windows service since the variable can change and the service not +%% be reinstalled, so in that case we add a magic application env. +config_setting() -> + case application:get_env(rabbit, windows_service_config) of + {ok, File1} -> File1; + undefined -> case os:getenv("RABBITMQ_CONFIG_FILE") of + false -> none; + File2 -> File2 + end + end. + diff --git a/src/rabbit_connection_tracking.erl b/src/rabbit_connection_tracking.erl new file mode 100644 index 0000000000..460bf964a0 --- /dev/null +++ b/src/rabbit_connection_tracking.erl @@ -0,0 +1,319 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_connection_tracking). + +%% Abstracts away how tracked connection records are stored +%% and queried. +%% +%% See also: +%% +%% * rabbit_connection_tracking_handler +%% * rabbit_reader +%% * rabbit_event + +-export([boot/0, + ensure_tracked_connections_table_for_node/1, + ensure_per_vhost_tracked_connections_table_for_node/1, + ensure_tracked_connections_table_for_this_node/0, + ensure_per_vhost_tracked_connections_table_for_this_node/0, + tracked_connection_table_name_for/1, tracked_connection_per_vhost_table_name_for/1, + delete_tracked_connections_table_for_node/1, delete_per_vhost_tracked_connections_table_for_node/1, + clear_tracked_connection_tables_for_this_node/0, + register_connection/1, unregister_connection/1, + list/0, list/1, list_on_node/1, + tracked_connection_from_connection_created/1, + tracked_connection_from_connection_state/1, + count_connections_in/1]). + +-include_lib("rabbit.hrl"). + +-import(rabbit_misc, [pget/2]). + +%% +%% API +%% + +-spec boot() -> ok. + +%% Sets up and resets connection tracking tables for this +%% node. +boot() -> + ensure_tracked_connections_table_for_this_node(), + rabbit_log:info("Setting up a table for connection tracking on this node: ~p", + [tracked_connection_table_name_for(node())]), + ensure_per_vhost_tracked_connections_table_for_this_node(), + rabbit_log:info("Setting up a table for per-vhost connection counting on this node: ~p", + [tracked_connection_per_vhost_table_name_for(node())]), + clear_tracked_connection_tables_for_this_node(), + ok. + + +-spec ensure_tracked_connections_table_for_this_node() -> ok. + +ensure_tracked_connections_table_for_this_node() -> + ensure_tracked_connections_table_for_node(node()). + + +-spec ensure_per_vhost_tracked_connections_table_for_this_node() -> ok. + +ensure_per_vhost_tracked_connections_table_for_this_node() -> + ensure_per_vhost_tracked_connections_table_for_node(node()). + + +-spec ensure_tracked_connections_table_for_node(node()) -> ok. + +ensure_tracked_connections_table_for_node(Node) -> + TableName = tracked_connection_table_name_for(Node), + case mnesia:create_table(TableName, [{record_name, tracked_connection}, + {attributes, record_info(fields, tracked_connection)}]) of + {atomic, ok} -> ok; + {aborted, {already_exists, _}} -> ok; + {aborted, Error} -> + rabbit_log:error("Failed to create a tracked connection table for node ~p: ~p", [Node, Error]), + ok + end. + + +-spec ensure_per_vhost_tracked_connections_table_for_node(node()) -> ok. + +ensure_per_vhost_tracked_connections_table_for_node(Node) -> + TableName = tracked_connection_per_vhost_table_name_for(Node), + case mnesia:create_table(TableName, [{record_name, tracked_connection_per_vhost}, + {attributes, record_info(fields, tracked_connection_per_vhost)}]) of + {atomic, ok} -> ok; + {aborted, {already_exists, _}} -> ok; + {aborted, Error} -> + rabbit_log:error("Failed to create a per-vhost tracked connection table for node ~p: ~p", [Node, Error]), + ok + end. + + +-spec clear_tracked_connection_tables_for_this_node() -> ok. + +clear_tracked_connection_tables_for_this_node() -> + case mnesia:clear_table(tracked_connection_table_name_for(node())) of + {atomic, ok} -> ok; + {aborted, _} -> ok + end, + case mnesia:clear_table(tracked_connection_per_vhost_table_name_for(node())) of + {atomic, ok} -> ok; + {aborted, _} -> ok + end. + + +-spec delete_tracked_connections_table_for_node(node()) -> ok. + +delete_tracked_connections_table_for_node(Node) -> + TableName = tracked_connection_table_name_for(Node), + case mnesia:delete_table(TableName) of + {atomic, ok} -> ok; + {aborted, {no_exists, _}} -> ok; + {aborted, Error} -> + rabbit_log:error("Failed to delete a tracked connection table for node ~p: ~p", [Node, Error]), + ok + end. + + +-spec delete_per_vhost_tracked_connections_table_for_node(node()) -> ok. + +delete_per_vhost_tracked_connections_table_for_node(Node) -> + TableName = tracked_connection_per_vhost_table_name_for(Node), + case mnesia:delete_table(TableName) of + {atomic, ok} -> ok; + {aborted, {no_exists, _}} -> ok; + {aborted, Error} -> + rabbit_log:error("Failed to delete a per-vhost tracked connection table for node ~p: ~p", [Node, Error]), + ok + end. + + +-spec tracked_connection_table_name_for(node()) -> atom(). + +tracked_connection_table_name_for(Node) -> + list_to_atom(rabbit_misc:format("tracked_connection_on_node_~s", [Node])). + +-spec tracked_connection_per_vhost_table_name_for(node()) -> atom(). + +tracked_connection_per_vhost_table_name_for(Node) -> + list_to_atom(rabbit_misc:format("tracked_connection_per_vhost_on_node_~s", [Node])). + + +-spec register_connection(rabbit_types:tracked_connection()) -> ok. + +register_connection(#tracked_connection{vhost = VHost, id = ConnId, node = Node} = Conn) when Node =:= node() -> + TableName = tracked_connection_table_name_for(Node), + PerVhostTableName = tracked_connection_per_vhost_table_name_for(Node), + rabbit_misc:execute_mnesia_transaction( + fun() -> + %% upsert + case mnesia:dirty_read(TableName, ConnId) of + [] -> + mnesia:write(TableName, Conn, write), + mnesia:dirty_update_counter( + PerVhostTableName, VHost, 1); + [_Row] -> + ok + end, + ok + end). + +-spec unregister_connection(rabbit_types:connection_name()) -> ok. + +unregister_connection(ConnId = {Node, _Name}) when Node =:= node() -> + TableName = tracked_connection_table_name_for(Node), + PerVhostTableName = tracked_connection_per_vhost_table_name_for(Node), + rabbit_misc:execute_mnesia_transaction( + fun() -> + case mnesia:dirty_read(TableName, ConnId) of + [] -> ok; + [Row] -> + mnesia:dirty_update_counter( + PerVhostTableName, Row#tracked_connection.vhost, -1), + mnesia:delete({TableName, ConnId}) + end + end). + + +-spec list() -> [rabbit_types:tracked_connection()]. + +list() -> + lists:foldl( + fun (Node, Acc) -> + Tab = tracked_connection_table_name_for(Node), + Acc ++ mnesia:dirty_match_object(Tab, #tracked_connection{_ = '_'}) + end, [], rabbit_mnesia:cluster_nodes(running)). + + +-spec list(rabbit_types:vhost()) -> [rabbit_types:tracked_connection()]. + +list(VHost) -> + lists:foldl( + fun (Node, Acc) -> + Tab = tracked_connection_table_name_for(Node), + Acc ++ mnesia:dirty_match_object(Tab, #tracked_connection{vhost = VHost, _ = '_'}) + end, [], rabbit_mnesia:cluster_nodes(running)). + + +-spec list_on_node(node()) -> [rabbit_types:tracked_connection()]. + +list_on_node(Node) -> + try mnesia:dirty_match_object( + tracked_connection_table_name_for(Node), + #tracked_connection{_ = '_'}) + catch exit:{aborted, {no_exists, _}} -> [] + end. + +-spec count_connections_in(rabbit_types:vhost()) -> non_neg_integer(). + +count_connections_in(VirtualHost) -> + lists:foldl(fun (Node, Acc) -> + Tab = tracked_connection_per_vhost_table_name_for(Node), + try + N = case mnesia:transaction( + fun() -> + case mnesia:dirty_read({Tab, VirtualHost}) of + [] -> 0; + [Val] -> Val#tracked_connection_per_vhost.connection_count + end + end) of + {atomic, Val} -> Val; + {aborted, _Reason} -> 0 + end, + Acc + N + catch _:Err -> + rabbit_log:error( + "Failed to fetch number of connections in vhost ~p on node ~p:~n~p~n", + [VirtualHost, Err, Node]), + Acc + end + end, 0, rabbit_mnesia:cluster_nodes(running)). + +%% Returns a #tracked_connection from connection_created +%% event details. +%% +%% @see rabbit_connection_tracking_handler. +tracked_connection_from_connection_created(EventDetails) -> + %% Example event: + %% + %% [{type,network}, + %% {pid,<0.329.0>}, + %% {name,<<"127.0.0.1:60998 -> 127.0.0.1:5672">>}, + %% {port,5672}, + %% {peer_port,60998}, + %% {host,{0,0,0,0,0,65535,32512,1}}, + %% {peer_host,{0,0,0,0,0,65535,32512,1}}, + %% {ssl,false}, + %% {peer_cert_subject,''}, + %% {peer_cert_issuer,''}, + %% {peer_cert_validity,''}, + %% {auth_mechanism,<<"PLAIN">>}, + %% {ssl_protocol,''}, + %% {ssl_key_exchange,''}, + %% {ssl_cipher,''}, + %% {ssl_hash,''}, + %% {protocol,{0,9,1}}, + %% {user,<<"guest">>}, + %% {vhost,<<"/">>}, + %% {timeout,14}, + %% {frame_max,131072}, + %% {channel_max,65535}, + %% {client_properties, + %% [{<<"capabilities">>,table, + %% [{<<"publisher_confirms">>,bool,true}, + %% {<<"consumer_cancel_notify">>,bool,true}, + %% {<<"exchange_exchange_bindings">>,bool,true}, + %% {<<"basic.nack">>,bool,true}, + %% {<<"connection.blocked">>,bool,true}, + %% {<<"authentication_failure_close">>,bool,true}]}, + %% {<<"product">>,longstr,<<"Bunny">>}, + %% {<<"platform">>,longstr, + %% <<"ruby 2.3.0p0 (2015-12-25 revision 53290) [x86_64-darwin15]">>}, + %% {<<"version">>,longstr,<<"2.3.0.pre">>}, + %% {<<"information">>,longstr, + %% <<"http://rubybunny.info">>}]}, + %% {connected_at,1453214290847}] + Name = pget(name, EventDetails), + Node = pget(node, EventDetails), + #tracked_connection{id = {Node, Name}, + name = Name, + node = Node, + vhost = pget(vhost, EventDetails), + username = pget(user, EventDetails), + connected_at = pget(connected_at, EventDetails), + pid = pget(pid, EventDetails), + type = pget(type, EventDetails), + peer_host = pget(peer_host, EventDetails), + peer_port = pget(peer_port, EventDetails)}. + +tracked_connection_from_connection_state(#connection{ + vhost = VHost, + connected_at = Ts, + peer_host = PeerHost, + peer_port = PeerPort, + user = Username, + name = Name + }) -> + tracked_connection_from_connection_created( + [{name, Name}, + {node, node()}, + {vhost, VHost}, + {user, Username}, + {connected_at, Ts}, + {pid, self()}, + {type, network}, + {peer_port, PeerPort}, + {peer_host, PeerHost}]). diff --git a/src/rabbit_connection_tracking_handler.erl b/src/rabbit_connection_tracking_handler.erl new file mode 100644 index 0000000000..598fe686c3 --- /dev/null +++ b/src/rabbit_connection_tracking_handler.erl @@ -0,0 +1,115 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_connection_tracking_handler). + +%% This module keeps track of connection creation and termination events +%% on its local node. The primary goal here is to decouple connection +%% tracking from rabbit_reader in rabbit_common. +%% +%% Events from other nodes are ignored. + +-behaviour(gen_event). + +-export([init/1, handle_call/2, handle_event/2, handle_info/2, + terminate/2, code_change/3]). + +-include_lib("rabbit.hrl"). +-import(rabbit_misc, [pget/2]). + +-rabbit_boot_step({?MODULE, + [{description, "connection tracking event handler"}, + {mfa, {gen_event, add_handler, + [rabbit_event, ?MODULE, []]}}, + {cleanup, {gen_event, delete_handler, + [rabbit_event, ?MODULE, []]}}, + {requires, [rabbit_event, rabbit_node_monitor]}, + {enables, recovery}]}). + + +%% +%% API +%% + +init([]) -> + {ok, []}. + +handle_event(#event{type = connection_created, props = Details}, State) -> + ThisNode = node(), + case pget(node, Details) of + ThisNode -> + rabbit_connection_tracking:register_connection( + rabbit_connection_tracking:tracked_connection_from_connection_created(Details) + ); + _OtherNode -> + %% ignore + ok + end, + {ok, State}; +handle_event(#event{type = connection_closed, props = Details}, State) -> + ThisNode = node(), + case pget(node, Details) of + ThisNode -> + %% [{name,<<"127.0.0.1:64078 -> 127.0.0.1:5672">>}, + %% {pid,<0.1774.0>}, + %% {node, rabbit@hostname}] + rabbit_connection_tracking:unregister_connection( + {pget(node, Details), + pget(name, Details)}); + _OtherNode -> + %% ignore + ok + end, + {ok, State}; +handle_event(#event{type = vhost_deleted, props = Details}, State) -> + VHost = pget(name, Details), + rabbit_log_connection:info("Closing all connections in vhost '~s' because it's being deleted", [VHost]), + [close_connection(Conn, rabbit_misc:format("vhost '~s' is deleted", [VHost])) + || Conn <- rabbit_connection_tracking:list(VHost)], + {ok, State}; +handle_event(#event{type = user_deleted, props = Details}, State) -> + _Username = pget(name, Details), + %% TODO: force close and unregister connections from + %% this user. Moved to rabbitmq/rabbitmq-server#628. + {ok, State}; +%% A node had been deleted from the cluster. +handle_event(#event{type = node_deleted, props = Details}, State) -> + Node = pget(node, Details), + rabbit_log_connection:info("Node '~s' was removed from the cluster, deleting its connection tracking tables...", [Node]), + rabbit_connection_tracking:delete_tracked_connections_table_for_node(Node), + rabbit_connection_tracking:delete_per_vhost_tracked_connections_table_for_node(Node), + {ok, State}; +handle_event(_Event, State) -> + {ok, State}. + +handle_call(_Request, State) -> + {ok, not_understood, State}. + +handle_info(_Info, State) -> + {ok, State}. + +terminate(_Arg, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +close_connection(#tracked_connection{pid = Pid, type = network}, Message) -> + rabbit_networking:close_connection(Pid, Message); +close_connection(#tracked_connection{pid = Pid, type = direct}, Message) -> + %% Do an RPC call to the node running the direct client. + Node = node(Pid), + rpc:call(Node, amqp_direct_connection, server_close, [Pid, 320, Message]). diff --git a/src/rabbit_control_main.erl b/src/rabbit_control_main.erl index 8c245892b7..1619f25494 100644 --- a/src/rabbit_control_main.erl +++ b/src/rabbit_control_main.erl @@ -23,7 +23,7 @@ sync_queue/1, cancel_sync_queue/1, become/1, purge_queue/1]). --import(rabbit_misc, [rpc_call/4, rpc_call/5, rpc_call/7]). +-import(rabbit_misc, [rpc_call/4, rpc_call/5]). -define(EXTERNAL_CHECK_INTERVAL, 1000). @@ -72,8 +72,13 @@ {set_policy, [?VHOST_DEF, ?PRIORITY_DEF, ?APPLY_TO_DEF]}, {clear_policy, [?VHOST_DEF]}, + {set_operator_policy, [?VHOST_DEF, ?PRIORITY_DEF, ?APPLY_TO_DEF]}, + {clear_operator_policy, [?VHOST_DEF]}, {list_policies, [?VHOST_DEF]}, + {list_operator_policies, [?VHOST_DEF]}, + {set_vhost_limits, [?VHOST_DEF]}, + {clear_vhost_limits, [?VHOST_DEF]}, {list_queues, [?VHOST_DEF, ?OFFLINE_DEF, ?ONLINE_DEF, ?LOCAL_DEF]}, {list_exchanges, [?VHOST_DEF]}, {list_bindings, [?VHOST_DEF]}, @@ -286,14 +291,14 @@ action(start_app, Node, [], _Opts, Inform) -> action(reset, Node, [], _Opts, Inform) -> Inform("Resetting node ~p", [Node]), - require_mnesia_stopped(Node, + require_mnesia_stopped(Node, fun() -> call(Node, {rabbit_mnesia, reset, []}) end); action(force_reset, Node, [], _Opts, Inform) -> Inform("Forcefully resetting node ~p", [Node]), - require_mnesia_stopped(Node, + require_mnesia_stopped(Node, fun() -> call(Node, {rabbit_mnesia, force_reset, []}) end); @@ -305,21 +310,21 @@ action(join_cluster, Node, [ClusterNodeS], Opts, Inform) -> false -> disc end, Inform("Clustering node ~p with ~p", [Node, ClusterNode]), - require_mnesia_stopped(Node, + require_mnesia_stopped(Node, fun() -> rpc_call(Node, rabbit_mnesia, join_cluster, [ClusterNode, NodeType]) end); action(change_cluster_node_type, Node, ["ram"], _Opts, Inform) -> Inform("Turning ~p into a ram node", [Node]), - require_mnesia_stopped(Node, + require_mnesia_stopped(Node, fun() -> rpc_call(Node, rabbit_mnesia, change_cluster_node_type, [ram]) end); action(change_cluster_node_type, Node, [Type], _Opts, Inform) when Type =:= "disc" orelse Type =:= "disk" -> Inform("Turning ~p into a disc node", [Node]), - require_mnesia_stopped(Node, + require_mnesia_stopped(Node, fun() -> rpc_call(Node, rabbit_mnesia, change_cluster_node_type, [disc]) end); @@ -327,7 +332,7 @@ action(change_cluster_node_type, Node, [Type], _Opts, Inform) action(update_cluster_nodes, Node, [ClusterNodeS], _Opts, Inform) -> ClusterNode = list_to_atom(ClusterNodeS), Inform("Updating cluster nodes for ~p from ~p", [Node, ClusterNode]), - require_mnesia_stopped(Node, + require_mnesia_stopped(Node, fun() -> rpc_call(Node, rabbit_mnesia, update_cluster_nodes, [ClusterNode]) end); @@ -392,11 +397,8 @@ action(environment, Node, _App, _Opts, Inform) -> display_call_result(Node, {rabbit, environment, []}); action(rotate_logs, Node, [], _Opts, Inform) -> - Inform("Reopening logs for node ~p", [Node]), - call(Node, {rabbit, rotate_logs, [""]}); -action(rotate_logs, Node, Args = [Suffix], _Opts, Inform) -> - Inform("Rotating logs to files with suffix \"~s\"", [Suffix]), - call(Node, {rabbit, rotate_logs, Args}); + Inform("Rotating logs for node ~p", [Node]), + call(Node, {rabbit, rotate_logs, []}); action(hipe_compile, _Node, [TargetDir], _Opts, _Inform) -> ok = application:load(rabbit), @@ -492,9 +494,9 @@ action(set_disk_free_limit, Node, ["mem_relative", Arg], _Opts, Inform) -> _ -> Arg end), Inform("Setting disk free limit on ~p to ~p of total RAM", [Node, Frac]), - rpc_call(Node, - rabbit_disk_monitor, - set_disk_free_limit, + rpc_call(Node, + rabbit_disk_monitor, + set_disk_free_limit, [{mem_relative, Frac}]); @@ -548,6 +550,38 @@ action(clear_policy, Node, [Key], Opts, Inform) -> Inform("Clearing policy ~p", [Key]), rpc_call(Node, rabbit_policy, delete, [VHostArg, list_to_binary(Key)]); +action(set_operator_policy, Node, [Key, Pattern, Defn], Opts, Inform) -> + Msg = "Setting operator policy override ~p for pattern ~p to ~p with priority ~p", + VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), + PriorityArg = proplists:get_value(?PRIORITY_OPT, Opts), + ApplyToArg = list_to_binary(proplists:get_value(?APPLY_TO_OPT, Opts)), + Inform(Msg, [Key, Pattern, Defn, PriorityArg]), + Res = rpc_call( + Node, rabbit_policy, parse_set_op, + [VHostArg, list_to_binary(Key), Pattern, Defn, PriorityArg, ApplyToArg]), + case Res of + {error, Format, Args} when is_list(Format) andalso is_list(Args) -> + {error_string, rabbit_misc:format(Format, Args)}; + _ -> + Res + end; + +action(clear_operator_policy, Node, [Key], Opts, Inform) -> + VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), + Inform("Clearing operator policy ~p", [Key]), + rpc_call(Node, rabbit_policy, delete_op, [VHostArg, list_to_binary(Key)]); + +action(set_vhost_limits, Node, [Defn], Opts, Inform) -> + VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), + Inform("Setting vhost limits for vhost ~p", [VHostArg]), + rpc_call(Node, rabbit_vhost_limit, parse_set, [VHostArg, Defn]), + ok; + +action(clear_vhost_limits, Node, [], Opts, Inform) -> + VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), + Inform("Clearing vhost ~p limits", [VHostArg]), + rpc_call(Node, rabbit_vhost_limit, clear, [VHostArg]); + action(report, Node, _Args, _Opts, Inform) -> Inform("Reporting server status on ~p~n~n", [erlang:universaltime()]), [begin ok = action(Action, N, [], [], Inform), io:nl() end || @@ -607,41 +641,53 @@ action(purge_queue, Node, [Q], Opts, Inform, Timeout) -> action(list_users, Node, [], _Opts, Inform, Timeout) -> Inform("Listing users", []), - call(Node, {rabbit_auth_backend_internal, list_users, []}, - rabbit_auth_backend_internal:user_info_keys(), true, Timeout); + call_emitter(Node, {rabbit_auth_backend_internal, list_users, []}, + rabbit_auth_backend_internal:user_info_keys(), + [{timeout, Timeout}, to_bin_utf8]); action(list_permissions, Node, [], Opts, Inform, Timeout) -> VHost = proplists:get_value(?VHOST_OPT, Opts), Inform("Listing permissions in vhost \"~s\"", [VHost]), - call(Node, {rabbit_auth_backend_internal, list_vhost_permissions, [VHost]}, - rabbit_auth_backend_internal:vhost_perms_info_keys(), true, Timeout, - true); + call_emitter(Node, {rabbit_auth_backend_internal, list_vhost_permissions, [VHost]}, + rabbit_auth_backend_internal:vhost_perms_info_keys(), + [{timeout, Timeout}, to_bin_utf8, is_escaped]); action(list_parameters, Node, [], Opts, Inform, Timeout) -> VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), Inform("Listing runtime parameters", []), - call(Node, {rabbit_runtime_parameters, list_formatted, [VHostArg]}, - rabbit_runtime_parameters:info_keys(), Timeout); + call_emitter(Node, {rabbit_runtime_parameters, list_formatted, [VHostArg]}, + rabbit_runtime_parameters:info_keys(), + [{timeout, Timeout}]); action(list_policies, Node, [], Opts, Inform, Timeout) -> VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), Inform("Listing policies", []), - call(Node, {rabbit_policy, list_formatted, [VHostArg]}, - rabbit_policy:info_keys(), Timeout); + call_emitter(Node, {rabbit_policy, list_formatted, [VHostArg]}, + rabbit_policy:info_keys(), + [{timeout, Timeout}]); + +action(list_operator_policies, Node, [], Opts, Inform, Timeout) -> + VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), + Inform("Listing policies", []), + call_emitter(Node, {rabbit_policy, list_formatted_op, [VHostArg]}, + rabbit_policy:info_keys(), + [{timeout, Timeout}]); + action(list_vhosts, Node, Args, _Opts, Inform, Timeout) -> Inform("Listing vhosts", []), ArgAtoms = default_if_empty(Args, [name]), - call(Node, {rabbit_vhost, info_all, []}, ArgAtoms, true, Timeout); + call_emitter(Node, {rabbit_vhost, info_all, []}, ArgAtoms, + [{timeout, Timeout}, to_bin_utf8]); action(list_user_permissions, _Node, _Args = [], _Opts, _Inform, _Timeout) -> {error_string, "list_user_permissions expects a username argument, but none provided."}; action(list_user_permissions, Node, Args = [_Username], _Opts, Inform, Timeout) -> Inform("Listing permissions for user ~p", Args), - call(Node, {rabbit_auth_backend_internal, list_user_permissions, Args}, - rabbit_auth_backend_internal:user_perms_info_keys(), true, Timeout, - true); + call_emitter(Node, {rabbit_auth_backend_internal, list_user_permissions, Args}, + rabbit_auth_backend_internal:user_perms_info_keys(), + [{timeout, Timeout}, to_bin_utf8, is_escaped]); action(list_queues, Node, Args, Opts, Inform, Timeout) -> case rabbit_cli:mutually_exclusive_flags( @@ -652,8 +698,36 @@ action(list_queues, Node, Args, Opts, Inform, Timeout) -> Inform("Listing queues", []), VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), ArgAtoms = default_if_empty(Args, [name, messages]), - call(Node, {rabbit_amqqueue, info_all, [VHostArg, ArgAtoms, Filter]}, - ArgAtoms, Timeout); + + %% Data for emission + Nodes = nodes_in_cluster(Node, Timeout), + ChunksOpt = {chunks, get_number_of_chunks(Filter, Nodes)}, + TimeoutOpt = {timeout, Timeout}, + EmissionRef = make_ref(), + EmissionRefOpt = {ref, EmissionRef}, + + case Filter of + all -> + start_emission(Node, {rabbit_amqqueue, emit_info_all, + [Nodes, VHostArg, ArgAtoms]}, + [TimeoutOpt, EmissionRefOpt]), + start_emission(Node, {rabbit_amqqueue, emit_info_down, + [VHostArg, ArgAtoms]}, + [TimeoutOpt, EmissionRefOpt]); + online -> + start_emission(Node, {rabbit_amqqueue, emit_info_all, + [Nodes, VHostArg, ArgAtoms]}, + [TimeoutOpt, EmissionRefOpt]); + offline -> + start_emission(Node, {rabbit_amqqueue, emit_info_down, + [VHostArg, ArgAtoms]}, + [TimeoutOpt, EmissionRefOpt]); + local -> + start_emission(Node, {rabbit_amqqueue, emit_info_local, + [VHostArg, ArgAtoms]}, + [TimeoutOpt, EmissionRefOpt]) + end, + display_emission_result(EmissionRef, ArgAtoms, [ChunksOpt, TimeoutOpt]); {error, ErrStr} -> {error_string, ErrStr} end; @@ -662,8 +736,8 @@ action(list_exchanges, Node, Args, Opts, Inform, Timeout) -> Inform("Listing exchanges", []), VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), ArgAtoms = default_if_empty(Args, [name, type]), - call(Node, {rabbit_exchange, info_all, [VHostArg, ArgAtoms]}, - ArgAtoms, Timeout); + call_emitter(Node, {rabbit_exchange, info_all, [VHostArg, ArgAtoms]}, + ArgAtoms, [{timeout, Timeout}]); action(list_bindings, Node, Args, Opts, Inform, Timeout) -> Inform("Listing bindings", []), @@ -671,27 +745,31 @@ action(list_bindings, Node, Args, Opts, Inform, Timeout) -> ArgAtoms = default_if_empty(Args, [source_name, source_kind, destination_name, destination_kind, routing_key, arguments]), - call(Node, {rabbit_binding, info_all, [VHostArg, ArgAtoms]}, - ArgAtoms, Timeout); + call_emitter(Node, {rabbit_binding, info_all, [VHostArg, ArgAtoms]}, + ArgAtoms, [{timeout, Timeout}]); action(list_connections, Node, Args, _Opts, Inform, Timeout) -> Inform("Listing connections", []), ArgAtoms = default_if_empty(Args, [user, peer_host, peer_port, state]), - call(Node, {rabbit_networking, connection_info_all, [ArgAtoms]}, - ArgAtoms, Timeout); + Nodes = nodes_in_cluster(Node, Timeout), + call_emitter(Node, {rabbit_networking, emit_connection_info_all, [Nodes, ArgAtoms]}, + ArgAtoms, [{timeout, Timeout}, {chunks, length(Nodes)}]); action(list_channels, Node, Args, _Opts, Inform, Timeout) -> Inform("Listing channels", []), ArgAtoms = default_if_empty(Args, [pid, user, consumer_count, messages_unacknowledged]), - call(Node, {rabbit_channel, info_all, [ArgAtoms]}, - ArgAtoms, Timeout); + Nodes = nodes_in_cluster(Node, Timeout), + call_emitter(Node, {rabbit_channel, emit_info_all, [Nodes, ArgAtoms]}, ArgAtoms, + [{timeout, Timeout}, {chunks, length(Nodes)}]); action(list_consumers, Node, _Args, Opts, Inform, Timeout) -> Inform("Listing consumers", []), VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), - call(Node, {rabbit_amqqueue, consumers_all, [VHostArg]}, - rabbit_amqqueue:consumer_info_keys(), Timeout); + Nodes = nodes_in_cluster(Node, Timeout), + call_emitter(Node, {rabbit_amqqueue, emit_consumers_all, [Nodes, VHostArg]}, + rabbit_amqqueue:consumer_info_keys(), + [{timeout, Timeout}, {chunks, length(Nodes)}]); action(node_health_check, Node, _Args, _Opts, Inform, Timeout) -> Inform("Checking health of node ~p", [Node]), @@ -807,17 +885,18 @@ display_info_message_row(IsEscaped, Result, InfoItemKeys) -> {X, Value} -> Value end, IsEscaped) || X <- InfoItemKeys]). -display_info_message(IsEscaped) -> +display_info_message(IsEscaped, InfoItemKeys) -> fun ([], _) -> ok; - ([FirstResult|_] = List, InfoItemKeys) when is_list(FirstResult) -> + ([FirstResult|_] = List, _) when is_list(FirstResult) -> lists:foreach(fun(Result) -> display_info_message_row(IsEscaped, Result, InfoItemKeys) end, List), ok; - (Result, InfoItemKeys) -> - display_info_message_row(IsEscaped, Result, InfoItemKeys) + (Result, _) -> + display_info_message_row(IsEscaped, Result, InfoItemKeys), + ok end. display_info_list(Results, InfoItemKeys) when is_list(Results) -> @@ -863,6 +942,9 @@ format_info_item([T | _] = Value, IsEscaped) lists:nthtail(2, lists:append( [", " ++ format_info_item(E, IsEscaped) || E <- Value])) ++ "]"; +format_info_item({Key, Value}, IsEscaped) -> + "{" ++ io_lib:format("~p", [Key]) ++ ", " ++ + format_info_item(Value, IsEscaped) ++ "}"; format_info_item(Value, _IsEscaped) -> io_lib:format("~w", [Value]). @@ -874,7 +956,10 @@ display_call_result(Node, MFA) -> end. unsafe_rpc(Node, Mod, Fun, Args) -> - case rpc_call(Node, Mod, Fun, Args) of + unsafe_rpc(Node, Mod, Fun, Args, ?RPC_TIMEOUT). + +unsafe_rpc(Node, Mod, Fun, Args, Timeout) -> + case rpc_call(Node, Mod, Fun, Args, Timeout) of {badrpc, _} = Res -> throw(Res); Normal -> Normal end. @@ -893,33 +978,42 @@ ensure_app_running(Node) -> call(Node, {Mod, Fun, Args}) -> rpc_call(Node, Mod, Fun, lists:map(fun list_to_binary_utf8/1, Args)). -call(Node, {Mod, Fun, Args}, InfoKeys, Timeout) -> - call(Node, {Mod, Fun, Args}, InfoKeys, false, Timeout, false). +call_emitter(Node, {Mod, Fun, Args}, InfoKeys, Opts) -> + Ref = start_emission(Node, {Mod, Fun, Args}, Opts), + display_emission_result(Ref, InfoKeys, Opts). + +start_emission(Node, {Mod, Fun, Args}, Opts) -> + ToBinUtf8 = proplists:get_value(to_bin_utf8, Opts, false), + Timeout = proplists:get_value(timeout, Opts, infinity), + Ref = proplists:get_value(ref, Opts, make_ref()), + rabbit_control_misc:spawn_emitter_caller( + Node, Mod, Fun, prepare_call_args(Args, ToBinUtf8), + Ref, self(), Timeout), + Ref. + +display_emission_result(Ref, InfoKeys, Opts) -> + IsEscaped = proplists:get_value(is_escaped, Opts, false), + Chunks = proplists:get_value(chunks, Opts, 1), + Timeout = proplists:get_value(timeout, Opts, infinity), + EmissionStatus = rabbit_control_misc:wait_for_info_messages( + self(), Ref, display_info_message(IsEscaped, InfoKeys), ok, Timeout, Chunks), + emission_to_action_result(EmissionStatus). + +%% Convert rabbit_control_misc:wait_for_info_messages/6 return value +%% into form expected by rabbit_cli:main/3. +emission_to_action_result({ok, ok}) -> + ok; +emission_to_action_result({error, Error}) -> + Error. -call(Node, {Mod, Fun, Args}, InfoKeys, ToBinUtf8, Timeout) -> - call(Node, {Mod, Fun, Args}, InfoKeys, ToBinUtf8, Timeout, false). +prepare_call_args(Args, ToBinUtf8) -> + case ToBinUtf8 of + true -> valid_utf8_args(Args); + false -> Args + end. -call(Node, {Mod, Fun, Args}, InfoKeys, ToBinUtf8, Timeout, IsEscaped) -> - Args0 = case ToBinUtf8 of - true -> lists:map(fun list_to_binary_utf8/1, Args); - false -> Args - end, - Ref = make_ref(), - Pid = self(), - spawn_link( - fun () -> - case rabbit_cli:rpc_call(Node, Mod, Fun, Args0, - Ref, Pid, Timeout) of - {error, _} = Error -> - Pid ! {error, Error}; - {bad_argument, _} = Error -> - Pid ! {error, Error}; - _ -> - ok - end - end), - rabbit_control_misc:wait_for_info_messages( - Pid, Ref, InfoKeys, display_info_message(IsEscaped), Timeout). +valid_utf8_args(Args) -> + lists:map(fun list_to_binary_utf8/1, Args). list_to_binary_utf8(L) -> B = list_to_binary(L), @@ -939,7 +1033,7 @@ escape(Bin, IsEscaped) when is_binary(Bin) -> escape(L, false) when is_list(L) -> escape_char(lists:reverse(L), []); escape(L, true) when is_list(L) -> - L. + L. escape_char([$\\ | T], Acc) -> escape_char(T, [$\\, $\\ | Acc]); @@ -969,7 +1063,10 @@ split_list([_]) -> exit(even_list_needed); split_list([A, B | T]) -> [{A, B} | split_list(T)]. nodes_in_cluster(Node) -> - unsafe_rpc(Node, rabbit_mnesia, cluster_nodes, [running]). + unsafe_rpc(Node, rabbit_mnesia, cluster_nodes, [running], ?RPC_TIMEOUT). + +nodes_in_cluster(Node, Timeout) -> + unsafe_rpc(Node, rabbit_mnesia, cluster_nodes, [running], Timeout). alarms_by_node(Name) -> case rpc_call(Name, rabbit, status, []) of @@ -978,3 +1075,12 @@ alarms_by_node(Name) -> {_, As} = lists:keyfind(alarms, 1, Status), {Name, As} end. + +get_number_of_chunks(all, Nodes) -> + length(Nodes) + 1; +get_number_of_chunks(online, Nodes) -> + length(Nodes); +get_number_of_chunks(offline, _) -> + 1; +get_number_of_chunks(local, _) -> + 1. diff --git a/src/rabbit_dead_letter.erl b/src/rabbit_dead_letter.erl index 91d23c83a4..5b367e410a 100644 --- a/src/rabbit_dead_letter.erl +++ b/src/rabbit_dead_letter.erl @@ -49,7 +49,7 @@ make_msg(Msg = #basic_message{content = Content, _ -> {[RK], fun (H) -> lists:keydelete(<<"CC">>, 1, H) end} end, ReasonBin = list_to_binary(atom_to_list(Reason)), - TimeSec = time_compat:os_system_time(seconds), + TimeSec = os:system_time(seconds), PerMsgTTL = per_msg_ttl_header(Content#content.properties), HeadersFun2 = fun (Headers) -> diff --git a/src/rabbit_direct.erl b/src/rabbit_direct.erl index 061105c150..53b0340b8a 100644 --- a/src/rabbit_direct.erl +++ b/src/rabbit_direct.erl @@ -65,35 +65,64 @@ list() -> %%---------------------------------------------------------------------------- -connect({none, _}, VHost, Protocol, Pid, Infos) -> - connect0(fun () -> {ok, rabbit_auth_backend_dummy:user()} end, - VHost, Protocol, Pid, Infos); +auth_fun({none, _}, _VHost) -> + fun () -> {ok, rabbit_auth_backend_dummy:user()} end; -connect({Username, none}, VHost, Protocol, Pid, Infos) -> - connect0(fun () -> rabbit_access_control:check_user_login(Username, []) end, - VHost, Protocol, Pid, Infos); +auth_fun({Username, none}, _VHost) -> + fun () -> rabbit_access_control:check_user_login(Username, []) end; -connect({Username, Password}, VHost, Protocol, Pid, Infos) -> - connect0(fun () -> rabbit_access_control:check_user_login( - Username, [{password, Password}, {vhost, VHost}]) end, - VHost, Protocol, Pid, Infos). +auth_fun({Username, Password}, VHost) -> + fun () -> + rabbit_access_control:check_user_login( + Username, + [{password, Password}, {vhost, VHost}]) + end. -connect0(AuthFun, VHost, Protocol, Pid, Infos) -> +connect(Creds, VHost, Protocol, Pid, Infos) -> + AuthFun = auth_fun(Creds, VHost), case rabbit:is_running() of - true -> case AuthFun() of - {ok, User = #user{username = Username}} -> - notify_auth_result(Username, - user_authentication_success, []), - connect1(User, VHost, Protocol, Pid, Infos); - {refused, Username, Msg, Args} -> - notify_auth_result(Username, - user_authentication_failure, - [{error, rabbit_misc:format(Msg, Args)}]), - {error, {auth_failure, "Refused"}} - end; + true -> + case is_over_connection_limit(VHost, Creds, Pid) of + true -> + {error, not_allowed}; + false -> + case AuthFun() of + {ok, User = #user{username = Username}} -> + notify_auth_result(Username, + user_authentication_success, []), + connect1(User, VHost, Protocol, Pid, Infos); + {refused, Username, Msg, Args} -> + notify_auth_result(Username, + user_authentication_failure, + [{error, rabbit_misc:format(Msg, Args)}]), + {error, {auth_failure, "Refused"}} + end + end; false -> {error, broker_not_found_on_node} end. +is_over_connection_limit(VHost, {Username, _Password}, Pid) -> + PrintedUsername = case Username of + none -> ""; + _ -> Username + end, + try rabbit_vhost_limit:is_over_connection_limit(VHost) of + false -> false; + {true, Limit} -> + rabbit_log_connection:error( + "Error on Direct connection ~p~n" + "access to vhost '~s' refused for user '~s': " + "connection limit (~p) is reached", + [Pid, VHost, PrintedUsername, Limit]), + true + catch + throw:{error, {no_such_vhost, VHost}} -> + rabbit_log_connection:error( + "Error on Direct connection ~p~n" + "vhost ~s not found", [Pid, VHost]), + true + end. + notify_auth_result(Username, AuthResult, ExtraProps) -> EventProps = [{connection_type, direct}, {name, case Username of none -> ''; _ -> Username end}] ++ diff --git a/src/rabbit_error_logger.erl b/src/rabbit_error_logger.erl index 5ba3ce7a4f..a59afe6c43 100644 --- a/src/rabbit_error_logger.erl +++ b/src/rabbit_error_logger.erl @@ -27,7 +27,6 @@ -export([init/1, terminate/2, code_change/3, handle_call/2, handle_event/2, handle_info/2]). --import(rabbit_error_logger_file_h, [safe_handle_event/3]). %%---------------------------------------------------------------------------- @@ -97,7 +96,7 @@ publish(_Other, _Format, _Data, _State) -> publish1(RoutingKey, Format, Data, LogExch) -> %% 0-9-1 says the timestamp is a "64 bit POSIX timestamp". That's %% second resolution, not millisecond. - Timestamp = time_compat:os_system_time(seconds), + Timestamp = os:system_time(seconds), Args = [truncate:term(A, ?LOG_TRUNC) || A <- Data], Headers = [{<<"node">>, longstr, list_to_binary(atom_to_list(node()))}], @@ -106,6 +105,19 @@ publish1(RoutingKey, Format, Data, LogExch) -> timestamp = Timestamp, headers = Headers}, list_to_binary(io_lib:format(Format, Args))) of - {ok, _QPids} -> ok; - {error, _Err} -> ok + {ok, _DeliveredQPids} -> ok; + {error, not_found} -> ok + end. + + +safe_handle_event(HandleEvent, Event, State) -> + try + HandleEvent(Event, State) + catch + _:Error -> + io:format( + "Error in log handler~n====================~n" + "Event: ~P~nError: ~P~nStack trace: ~p~n~n", + [Event, 30, Error, 30, erlang:get_stacktrace()]), + {ok, State} end. diff --git a/src/rabbit_error_logger_file_h.erl b/src/rabbit_error_logger_file_h.erl deleted file mode 100644 index 930aead392..0000000000 --- a/src/rabbit_error_logger_file_h.erl +++ /dev/null @@ -1,180 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. -%% - --module(rabbit_error_logger_file_h). --include("rabbit.hrl"). - --behaviour(gen_event). - --export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2, - code_change/3]). - --export([safe_handle_event/3]). - -%% extracted from error_logger_file_h. Since 18.1 the state of the -%% error logger module changed. See: -%% https://github.com/erlang/otp/commit/003091a1fcc749a182505ef5675c763f71eacbb0#diff-d9a19ba08f5d2b60fadfc3aa1566b324R108 -%% github issue: -%% https://github.com/rabbitmq/rabbitmq-server/issues/324 --record(st, {fd, - filename, - prev_handler, - depth = unlimited}). - -%% extracted from error_logger_file_h. See comment above. -get_depth() -> - case application:get_env(kernel, error_logger_format_depth) of - {ok, Depth} when is_integer(Depth) -> - erlang:max(10, Depth); - undefined -> - unlimited - end. - --define(ERTS_NEW_LOGGER_STATE, "7.1"). - -%% rabbit_error_logger_file_h is a wrapper around the error_logger_file_h -%% module because the original's init/1 does not match properly -%% with the result of closing the old handler when swapping handlers. -%% The first init/1 additionally allows for simple log rotation -%% when the suffix is not the empty string. -%% The original init/2 also opened the file in 'write' mode, thus -%% overwriting old logs. To remedy this, init/2 from -%% lib/stdlib/src/error_logger_file_h.erl from R14B3 was copied as -%% init_file/2 and changed so that it opens the file in 'append' mode. - -%% Log rotation with empty suffix should result only in file re-opening. -init({{File, ""}, _}) -> - init(File); -%% Used only when swapping handlers in log rotation, pre OTP 18.1 -init({{File, Suffix}, []}) -> - rotate_logs(File, Suffix), - init(File); -%% Used only when swapping handlers in log rotation, since OTP 18.1 -init({{File, Suffix}, ok}) -> - rotate_logs(File, Suffix), - init(File); -%% Used only when swapping handlers and the original handler -%% failed to terminate or was never installed -init({{File, _}, error}) -> - init(File); -%% Used only when swapping handlers without performing -%% log rotation -init({File, []}) -> - init(File); -%% Used only when taking over from the tty handler -init({{File, []}, _}) -> - init(File); -init({File, {error_logger, Buf}}) -> - rabbit_file:ensure_parent_dirs_exist(File), - init_file(File, {error_logger, Buf}); -init(File) -> - rabbit_file:ensure_parent_dirs_exist(File), - init_file(File, []). - -init_file(File, {error_logger, Buf}) -> - case init_file(File, error_logger) of - {ok, State} -> - [handle_event(Event, State) || - {_, Event} <- lists:reverse(Buf)], - {ok, State}; - Error -> - Error - end; -init_file(File, PrevHandler) -> - process_flag(trap_exit, true), - case file:open(File, [append]) of - {ok, Fd} -> - FoundVer = erlang:system_info(version), - State = - case rabbit_misc:version_compare( - ?ERTS_NEW_LOGGER_STATE, FoundVer, lte) of - true -> - #st{fd = Fd, - filename = File, - prev_handler = PrevHandler, - depth = get_depth()}; - _ -> - {Fd, File, PrevHandler} - end, - {ok, State}; - Error -> Error - end. - -handle_event(Event, State) -> - safe_handle_event(fun handle_event0/2, Event, State). - -safe_handle_event(HandleEvent, Event, State) -> - try - HandleEvent(Event, State) - catch - _:Error -> - io:format( - "Error in log handler~n====================~n" - "Event: ~P~nError: ~P~nStack trace: ~p~n~n", - [Event, 30, Error, 30, erlang:get_stacktrace()]), - {ok, State} - end. - -%% filter out "application: foo; exited: stopped; type: temporary" -handle_event0({info_report, _, {_, std_info, _}}, State) -> - {ok, State}; -%% When a node restarts quickly it is possible the rest of the cluster -%% will not have had the chance to remove its queues from -%% Mnesia. That's why rabbit_amqqueue:recover/0 invokes -%% on_node_down(node()). But before we get there we can receive lots -%% of messages intended for the old version of the node. The emulator -%% logs an event for every one of those messages; in extremis this can -%% bring the server to its knees just logging "Discarding..." -%% again and again. So just log the first one, then go silent. -handle_event0(Event = {error, _, {emulator, _, ["Discarding message" ++ _]}}, - State) -> - case get(discarding_message_seen) of - true -> {ok, State}; - undefined -> put(discarding_message_seen, true), - error_logger_file_h:handle_event(t(Event), State) - end; -%% Clear this state if we log anything else (but not a progress report). -handle_event0(Event = {info_msg, _, _}, State) -> - erase(discarding_message_seen), - error_logger_file_h:handle_event(t(Event), State); -handle_event0(Event, State) -> - error_logger_file_h:handle_event(t(Event), State). - -handle_info(Info, State) -> - error_logger_file_h:handle_info(Info, State). - -handle_call(Call, State) -> - error_logger_file_h:handle_call(Call, State). - -terminate(Reason, State) -> - error_logger_file_h:terminate(Reason, State). - -code_change(OldVsn, State, Extra) -> - error_logger_file_h:code_change(OldVsn, State, Extra). - -%%---------------------------------------------------------------------- - -t(Term) -> truncate:log_event(Term, ?LOG_TRUNC). - -rotate_logs(File, Suffix) -> - case rabbit_file:append_file(File, Suffix) of - ok -> file:delete(File), - ok; - {error, Error} -> - rabbit_log:error("Failed to append contents of " - "log file '~s' to '~s':~n~p~n", - [File, [File, Suffix], Error]) - end. diff --git a/src/rabbit_exchange.erl b/src/rabbit_exchange.erl index aaea27f91a..252817fba1 100644 --- a/src/rabbit_exchange.erl +++ b/src/rabbit_exchange.erl @@ -342,11 +342,17 @@ i(policy, X) -> case rabbit_policy:name(X) of none -> ''; Policy -> Policy end; -i(Item, _) -> throw({bad_argument, Item}). +i(Item, #exchange{type = Type} = X) -> + case (type_to_module(Type)):info(X, [Item]) of + [{Item, I}] -> I; + [] -> throw({bad_argument, Item}) + end. -info(X = #exchange{}) -> infos(?INFO_KEYS, X). +info(X = #exchange{type = Type}) -> + infos(?INFO_KEYS, X) ++ (type_to_module(Type)):info(X). -info(X = #exchange{}, Items) -> infos(Items, X). +info(X = #exchange{type = _Type}, Items) -> + infos(Items, X). info_all(VHostPath) -> map(VHostPath, fun (X) -> info(X) end). diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index 8a6886e376..ed675b572a 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -23,6 +23,7 @@ -export([validate/1, validate_binding/2, create/2, delete/3, policy_changed/2, add_binding/3, remove_bindings/3, assert_args_equivalence/2]). +-export([info/1, info/2]). -rabbit_boot_step({?MODULE, [{description, "exchange type direct"}, @@ -31,6 +32,9 @@ {requires, rabbit_registry}, {enables, kernel_ready}]}). +info(_X) -> []. +info(_X, _) -> []. + description() -> [{description, <<"AMQP direct exchange, as per the AMQP specification">>}]. diff --git a/src/rabbit_exchange_type_fanout.erl b/src/rabbit_exchange_type_fanout.erl index d81e407f8f..3aebc07b41 100644 --- a/src/rabbit_exchange_type_fanout.erl +++ b/src/rabbit_exchange_type_fanout.erl @@ -23,6 +23,7 @@ -export([validate/1, validate_binding/2, create/2, delete/3, policy_changed/2, add_binding/3, remove_bindings/3, assert_args_equivalence/2]). +-export([info/1, info/2]). -rabbit_boot_step({?MODULE, [{description, "exchange type fanout"}, @@ -31,6 +32,9 @@ {requires, rabbit_registry}, {enables, kernel_ready}]}). +info(_X) -> []. +info(_X, _) -> []. + description() -> [{description, <<"AMQP fanout exchange, as per the AMQP specification">>}]. diff --git a/src/rabbit_exchange_type_headers.erl b/src/rabbit_exchange_type_headers.erl index 196873aa22..f7db35be2f 100644 --- a/src/rabbit_exchange_type_headers.erl +++ b/src/rabbit_exchange_type_headers.erl @@ -24,6 +24,7 @@ -export([validate/1, validate_binding/2, create/2, delete/3, policy_changed/2, add_binding/3, remove_bindings/3, assert_args_equivalence/2]). +-export([info/1, info/2]). -rabbit_boot_step({?MODULE, [{description, "exchange type headers"}, @@ -36,6 +37,9 @@ (rabbit_framing:amqp_table(), rabbit_framing:amqp_table()) -> boolean(). +info(_X) -> []. +info(_X, _) -> []. + description() -> [{description, <<"AMQP headers exchange, as per the AMQP specification">>}]. diff --git a/src/rabbit_exchange_type_invalid.erl b/src/rabbit_exchange_type_invalid.erl index 2510c8a241..9edb00ee49 100644 --- a/src/rabbit_exchange_type_invalid.erl +++ b/src/rabbit_exchange_type_invalid.erl @@ -23,6 +23,10 @@ -export([validate/1, validate_binding/2, create/2, delete/3, policy_changed/2, add_binding/3, remove_bindings/3, assert_args_equivalence/2]). +-export([info/1, info/2]). + +info(_X) -> []. +info(_X, _) -> []. description() -> [{description, diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 0eccb66cfd..60be070426 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -24,6 +24,7 @@ -export([validate/1, validate_binding/2, create/2, delete/3, policy_changed/2, add_binding/3, remove_bindings/3, assert_args_equivalence/2]). +-export([info/1, info/2]). -rabbit_boot_step({?MODULE, [{description, "exchange type topic"}, @@ -34,6 +35,9 @@ %%---------------------------------------------------------------------------- +info(_X) -> []. +info(_X, _) -> []. + description() -> [{description, <<"AMQP topic exchange, as per the AMQP specification">>}]. diff --git a/src/rabbit_file.erl b/src/rabbit_file.erl index 878b9da7a7..d8e215bfc5 100644 --- a/src/rabbit_file.erl +++ b/src/rabbit_file.erl @@ -23,6 +23,7 @@ -export([append_file/2, ensure_parent_dirs_exist/1]). -export([rename/2, delete/1, recursive_delete/1, recursive_copy/2]). -export([lock_file/1]). +-export([read_file_info/1]). -export([filename_as_a_directory/1]). -import(file_handle_cache, [with_handle/1, with_handle/2]). diff --git a/src/rabbit_hipe.erl b/src/rabbit_hipe.erl index d4597d4efc..6957d85cb4 100644 --- a/src/rabbit_hipe.erl +++ b/src/rabbit_hipe.erl @@ -79,7 +79,7 @@ do_hipe_compile(HipeModules, CompileFun) -> Count = length(HipeModules), io:format("~nHiPE compiling: |~s|~n |", [string:copies("-", Count)]), - T1 = time_compat:monotonic_time(), + T1 = erlang:monotonic_time(), %% We use code:get_object_code/1 below to get the beam binary, %% instead of letting hipe get it itself, because hipe:c/{1,2} %% expects the given filename to actually exist on disk: it does not @@ -99,8 +99,8 @@ do_hipe_compile(HipeModules, CompileFun) -> {'DOWN', MRef, process, _, normal} -> ok; {'DOWN', MRef, process, _, Reason} -> exit(Reason) end || {_Pid, MRef} <- PidMRefs], - T2 = time_compat:monotonic_time(), - Duration = time_compat:convert_time_unit(T2 - T1, native, seconds), + T2 = erlang:monotonic_time(), + Duration = erlang:convert_time_unit(T2 - T1, native, seconds), io:format("|~n~nCompiled ~B modules in ~Bs~n", [Count, Duration]), {ok, Count, Duration}. diff --git a/src/rabbit_lager.erl b/src/rabbit_lager.erl new file mode 100644 index 0000000000..8beee10846 --- /dev/null +++ b/src/rabbit_lager.erl @@ -0,0 +1,267 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2015 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_lager). + +-include("rabbit_log.hrl"). + +%% API +-export([start_logger/0, log_locations/0, fold_sinks/2]). + +start_logger() -> + application:stop(lager), + ensure_lager_configured(), + lager:start(), + fold_sinks( + fun + (_, [], Acc) -> + Acc; + (SinkName, _, Acc) -> + lager:log(SinkName, info, self(), + "Log file opened with Lager", []), + Acc + end, ok), + ensure_log_working(). + +log_locations() -> + ensure_lager_configured(), + DefaultHandlers = application:get_env(lager, handlers, []), + Sinks = application:get_env(lager, extra_sinks, []), + ExtraHandlers = [proplists:get_value(handlers, Props, []) + || {_, Props} <- Sinks], + lists:sort(log_locations1([DefaultHandlers | ExtraHandlers], [])). + +log_locations1([Handlers | Rest], Locations) -> + Locations1 = log_locations2(Handlers, Locations), + log_locations1(Rest, Locations1); +log_locations1([], Locations) -> + Locations. + +log_locations2([{lager_file_backend, Settings} | Rest], Locations) -> + FileName = lager_file_name1(Settings), + Locations1 = case lists:member(FileName, Locations) of + false -> [FileName | Locations]; + true -> Locations + end, + log_locations2(Rest, Locations1); +log_locations2([{lager_console_backend, _} | Rest], Locations) -> + Locations1 = case lists:member("<stdout>", Locations) of + false -> ["<stdout>" | Locations]; + true -> Locations + end, + log_locations2(Rest, Locations1); +log_locations2([_ | Rest], Locations) -> + log_locations2(Rest, Locations); +log_locations2([], Locations) -> + Locations. + +fold_sinks(Fun, Acc) -> + Handlers = lager_config:global_get(handlers), + Sinks = dict:to_list(lists:foldl( + fun + ({{lager_file_backend, F}, _, S}, Dict) -> + dict:append(S, F, Dict); + ({_, _, S}, Dict) -> + case dict:is_key(S, Dict) of + true -> dict:store(S, [], Dict); + false -> Dict + end + end, + dict:new(), Handlers)), + fold_sinks(Sinks, Fun, Acc). + +fold_sinks([{SinkName, FileNames} | Rest], Fun, Acc) -> + Acc1 = Fun(SinkName, FileNames, Acc), + fold_sinks(Rest, Fun, Acc1); +fold_sinks([], _, Acc) -> + Acc. + +ensure_log_working() -> + {ok, Handlers} = application:get_env(lager, handlers), + [ ensure_lager_handler_file_exist(Handler) + || Handler <- Handlers ], + Sinks = application:get_env(lager, extra_sinks, []), + ensure_extra_sinks_working(Sinks, list_expected_sinks()). + +ensure_extra_sinks_working(Sinks, [SinkName | Rest]) -> + case proplists:get_value(SinkName, Sinks) of + undefined -> throw({error, {cannot_log_to_file, unknown, + rabbit_log_lager_event_sink_undefined}}); + Sink -> + SinkHandlers = proplists:get_value(handlers, Sink, []), + [ ensure_lager_handler_file_exist(Handler) + || Handler <- SinkHandlers ] + end, + ensure_extra_sinks_working(Sinks, Rest); +ensure_extra_sinks_working(_Sinks, []) -> + ok. + +ensure_lager_handler_file_exist(Handler) -> + case lager_file_name(Handler) of + false -> ok; + FileName -> ensure_logfile_exist(FileName) + end. + +lager_file_name({lager_file_backend, Settings}) -> + lager_file_name1(Settings); +lager_file_name(_) -> + false. + +lager_file_name1(Settings) when is_list(Settings) -> + {file, FileName} = proplists:lookup(file, Settings), + lager_util:expand_path(FileName); +lager_file_name1({FileName, _}) -> lager_util:expand_path(FileName); +lager_file_name1({FileName, _, _, _, _}) -> lager_util:expand_path(FileName); +lager_file_name1(_) -> + throw({error, {cannot_log_to_file, unknown, + lager_file_backend_config_invalid}}). + + +ensure_logfile_exist(FileName) -> + LogFile = lager_util:expand_path(FileName), + case rabbit_file:read_file_info(LogFile) of + {ok,_} -> ok; + {error, Err} -> throw({error, {cannot_log_to_file, LogFile, Err}}) + end. + +lager_handlers(Silent) when Silent == silent; Silent == false -> + []; +lager_handlers(tty) -> + [{lager_console_backend, info}]; +lager_handlers(FileName) when is_list(FileName) -> + LogFile = lager_util:expand_path(FileName), + case rabbit_file:ensure_dir(LogFile) of + ok -> ok; + {error, Reason} -> + throw({error, {cannot_log_to_file, LogFile, + {cannot_create_parent_dirs, LogFile, Reason}}}) + end, + [{lager_file_backend, [{file, FileName}, + {level, info}, + {formatter_config, + [date, " ", time, " ", color, "[", severity, "] ", + {pid, ""}, + " ", message, "\n"]}, + {date, ""}, + {size, 0}]}]. + + +ensure_lager_configured() -> + case lager_configured() of + false -> configure_lager(); + true -> ok + end. + +%% Lager should have handlers and sinks +lager_configured() -> + Sinks = lager:list_all_sinks(), + ExpectedSinks = list_expected_sinks(), + application:get_env(lager, handlers) =/= undefined + andalso + lists:all(fun(S) -> lists:member(S, Sinks) end, ExpectedSinks). + +configure_lager() -> + application:load(lager), + %% Turn off reformatting for error_logger messages + case application:get_env(lager, error_logger_format_raw) of + undefined -> application:set_env(lager, error_logger_format_raw, true); + _ -> ok + end, + case application:get_env(lager, log_root) of + undefined -> + application:set_env(lager, log_root, + application:get_env(rabbit, lager_log_root, + undefined)); + _ -> ok + end, + + %% Configure the default sink/handlers. + Handlers0 = application:get_env(lager, handlers, undefined), + DefaultHandlers = lager_handlers(application:get_env(rabbit, + lager_handler, + tty)), + Handlers = case os:getenv("RABBITMQ_LOGS_source") of + %% There are no default handlers in rabbitmq.config, create a + %% configuration from $RABBITMQ_LOGS. + false when Handlers0 =:= undefined -> DefaultHandlers; + %% There are default handlers configure in rabbitmq.config, but + %% the user explicitely sets $RABBITMQ_LOGS; create new handlers + %% based on that, instead of using rabbitmq.config. + "environment" -> DefaultHandlers; + %% Use the default handlers configured in rabbitmq.com. + _ -> Handlers0 + end, + application:set_env(lager, handlers, Handlers), + + %% Setup extra sink/handlers. If they are not configured, redirect + %% messages to the default sink. To know the list of expected extra + %% sinks, we look at the 'lager_extra_sinks' compilation option. + Sinks0 = application:get_env(lager, extra_sinks, []), + Sinks1 = configure_extra_sinks(Sinks0, + [error_logger | list_expected_sinks()]), + %% TODO Waiting for basho/lager#303 + %% Sinks2 = lists:keystore(error_logger_lager_event, 1, Sinks1, + %% {error_logger_lager_event, + %% [{handlers, Handlers}]}), + application:set_env(lager, extra_sinks, Sinks1), + + case application:get_env(lager, error_logger_hwm) of + undefined -> + application:set_env(lager, error_logger_hwm, 100); + {ok, Val} when is_integer(Val) andalso Val =< 100 -> + application:set_env(lager, error_logger_hwm, 100); + {ok, _Val} -> + ok + end, + ok. + +configure_extra_sinks(Sinks, [SinkName | Rest]) -> + Sink0 = proplists:get_value(SinkName, Sinks, []), + Sink1 = case proplists:is_defined(handlers, Sink0) of + false -> lists:keystore(handlers, 1, Sink0, + {handlers, + [{lager_forwarder_backend, + lager_util:make_internal_sink_name(lager) + }]}); + true -> Sink0 + end, + Sinks1 = lists:keystore(SinkName, 1, Sinks, {SinkName, Sink1}), + configure_extra_sinks(Sinks1, Rest); +configure_extra_sinks(Sinks, []) -> + Sinks. + +list_expected_sinks() -> + case application:get_env(rabbit, lager_extra_sinks) of + {ok, List} -> + List; + undefined -> + CompileOptions = proplists:get_value(options, + ?MODULE:module_info(compile), + []), + AutoList = [lager_util:make_internal_sink_name(M) + || M <- proplists:get_value(lager_extra_sinks, + CompileOptions, [])], + List = case lists:member(?LAGER_SINK, AutoList) of + true -> AutoList; + false -> [?LAGER_SINK | AutoList] + end, + %% Store the list in the application environment. If this + %% module is later cover-compiled, the compile option will + %% be lost, so we will be able to retrieve the list from the + %% application environment. + application:set_env(rabbit, lager_extra_sinks, List), + List + end. diff --git a/src/rabbit_limiter.erl b/src/rabbit_limiter.erl index 203e309b02..74e802b3a0 100644 --- a/src/rabbit_limiter.erl +++ b/src/rabbit_limiter.erl @@ -432,7 +432,7 @@ notify_queues(State = #lim{ch_pid = ChPid, queues = Queues}) -> %% We randomly vary the position of queues in the list, %% thus ensuring that each queue has an equal chance of %% being notified first. - {L1, L2} = lists:split(rand_compat:uniform(L), QList), + {L1, L2} = lists:split(rand:uniform(L), QList), [[ok = rabbit_amqqueue:resume(Q, ChPid) || Q <- L3] || L3 <- [L2, L1]], ok diff --git a/src/rabbit_log.erl b/src/rabbit_log.erl index 337fb23f84..f60cf6c0c2 100644 --- a/src/rabbit_log.erl +++ b/src/rabbit_log.erl @@ -16,96 +16,113 @@ -module(rabbit_log). --export([log/3, log/4, debug/1, debug/2, info/1, info/2, warning/1, - warning/2, error/1, error/2]). --export([with_local_io/1]). - +-export([log/3, log/4]). +-export([debug/1, debug/2, debug/3, + info/1, info/2, info/3, + notice/1, notice/2, notice/3, + warning/1, warning/2, warning/3, + error/1, error/2, error/3, + critical/1, critical/2, critical/3, + alert/1, alert/2, alert/3, + emergency/1, emergency/2, emergency/3, + none/1, none/2, none/3]). + +-include("rabbit_log.hrl"). %%---------------------------------------------------------------------------- --export_type([level/0]). - -type category() :: atom(). --type level() :: 'debug' | 'info' | 'warning' | 'error'. --spec log(category(), level(), string()) -> 'ok'. --spec log(category(), level(), string(), [any()]) -> 'ok'. +-spec log(category(), lager:log_level(), string()) -> 'ok'. +-spec log(category(), lager:log_level(), string(), [any()]) -> 'ok'. -spec debug(string()) -> 'ok'. -spec debug(string(), [any()]) -> 'ok'. +-spec debug(pid() | [tuple()], string(), [any()]) -> 'ok'. -spec info(string()) -> 'ok'. -spec info(string(), [any()]) -> 'ok'. +-spec info(pid() | [tuple()], string(), [any()]) -> 'ok'. +-spec notice(string()) -> 'ok'. +-spec notice(string(), [any()]) -> 'ok'. +-spec notice(pid() | [tuple()], string(), [any()]) -> 'ok'. -spec warning(string()) -> 'ok'. -spec warning(string(), [any()]) -> 'ok'. +-spec warning(pid() | [tuple()], string(), [any()]) -> 'ok'. -spec error(string()) -> 'ok'. -spec error(string(), [any()]) -> 'ok'. - --spec with_local_io(fun (() -> A)) -> A. +-spec error(pid() | [tuple()], string(), [any()]) -> 'ok'. +-spec critical(string()) -> 'ok'. +-spec critical(string(), [any()]) -> 'ok'. +-spec critical(pid() | [tuple()], string(), [any()]) -> 'ok'. +-spec alert(string()) -> 'ok'. +-spec alert(string(), [any()]) -> 'ok'. +-spec alert(pid() | [tuple()], string(), [any()]) -> 'ok'. +-spec emergency(string()) -> 'ok'. +-spec emergency(string(), [any()]) -> 'ok'. +-spec emergency(pid() | [tuple()], string(), [any()]) -> 'ok'. +-spec none(string()) -> 'ok'. +-spec none(string(), [any()]) -> 'ok'. +-spec none(pid() | [tuple()], string(), [any()]) -> 'ok'. %%---------------------------------------------------------------------------- log(Category, Level, Fmt) -> log(Category, Level, Fmt, []). log(Category, Level, Fmt, Args) when is_list(Args) -> - case level(Level) =< catlevel(Category) of - false -> ok; - true -> F = case Level of - debug -> fun error_logger:info_msg/2; - info -> fun error_logger:info_msg/2; - warning -> fun error_logger:warning_msg/2; - error -> fun error_logger:error_msg/2 - end, - with_local_io(fun () -> F(Fmt, Args) end) - end. - -debug(Fmt) -> log(default, debug, Fmt). -debug(Fmt, Args) -> log(default, debug, Fmt, Args). -info(Fmt) -> log(default, info, Fmt). -info(Fmt, Args) -> log(default, info, Fmt, Args). -warning(Fmt) -> log(default, warning, Fmt). -warning(Fmt, Args) -> log(default, warning, Fmt, Args). -error(Fmt) -> log(default, error, Fmt). -error(Fmt, Args) -> log(default, error, Fmt, Args). - -catlevel(Category) -> - %% We can get here as part of rabbitmqctl when it is impersonating - %% a node; in which case the env will not be defined. - CatLevelList = case application:get_env(rabbit, log_levels) of - {ok, L} -> L; - undefined -> [] - end, - level(proplists:get_value(Category, CatLevelList, info)). - -%%-------------------------------------------------------------------- - -level(debug) -> 4; -level(info) -> 3; -level(warning) -> 2; -level(warn) -> 2; -level(error) -> 1; -level(none) -> 0. - -%% Execute Fun using the IO system of the local node (i.e. the node on -%% which the code is executing). Since this is invoked for every log -%% message, we try to avoid unnecessarily churning group_leader/1. -with_local_io(Fun) -> - GL = group_leader(), - Node = node(), - case node(GL) of - Node -> Fun(); - _ -> set_group_leader_to_user_safely(whereis(user)), - try - Fun() - after - group_leader(GL, self()) - end - end. - -set_group_leader_to_user_safely(undefined) -> - handle_damaged_io_system(); -set_group_leader_to_user_safely(User) when is_pid(User) -> - group_leader(User, self()). - -handle_damaged_io_system() -> - Msg = "Erlang VM I/O system is damaged, restart needed~n", - io:format(standard_error, Msg, []), - exit(erlang_vm_restart_needed). + Sink = case Category of + default -> ?LAGER_SINK; + _ -> make_internal_sink_name(Category) + end, + lager:log(Sink, Level, self(), Fmt, Args). + +make_internal_sink_name(rabbit_log_connection) -> rabbit_log_connection_lager_event; +make_internal_sink_name(rabbit_log_channel) -> rabbit_log_channel_lager_event; +make_internal_sink_name(rabbit_log_mirroring) -> rabbit_log_mirroring_lager_event; +make_internal_sink_name(rabbit_log_queue) -> rabbit_log_queue_lager_event; +make_internal_sink_name(rabbit_log_federation) -> rabbit_log_federation_lager_event; +make_internal_sink_name(Category) -> + lager_util:make_internal_sink_name(Category). + +debug(Format) -> debug(Format, []). +debug(Format, Args) -> debug(self(), Format, Args). +debug(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, debug, Metadata, Format, Args). + +info(Format) -> info(Format, []). +info(Format, Args) -> info(self(), Format, Args). +info(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, info, Metadata, Format, Args). + +notice(Format) -> notice(Format, []). +notice(Format, Args) -> notice(self(), Format, Args). +notice(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, notice, Metadata, Format, Args). + +warning(Format) -> warning(Format, []). +warning(Format, Args) -> warning(self(), Format, Args). +warning(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, warning, Metadata, Format, Args). + +error(Format) -> ?MODULE:error(Format, []). +error(Format, Args) -> ?MODULE:error(self(), Format, Args). +error(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, error, Metadata, Format, Args). + +critical(Format) -> critical(Format, []). +critical(Format, Args) -> critical(self(), Format, Args). +critical(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, critical, Metadata, Format, Args). + +alert(Format) -> alert(Format, []). +alert(Format, Args) -> alert(self(), Format, Args). +alert(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, alert, Metadata, Format, Args). + +emergency(Format) -> emergency(Format, []). +emergency(Format, Args) -> emergency(self(), Format, Args). +emergency(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, emergency, Metadata, Format, Args). + +none(Format) -> none(Format, []). +none(Format, Args) -> none(self(), Format, Args). +none(Metadata, Format, Args) -> + lager:log(?LAGER_SINK, none, Metadata, Format, Args). diff --git a/src/rabbit_mirror_queue_master.erl b/src/rabbit_mirror_queue_master.erl index d82cdf336a..b006e37eb2 100644 --- a/src/rabbit_mirror_queue_master.erl +++ b/src/rabbit_mirror_queue_master.erl @@ -57,13 +57,13 @@ coordinator :: pid(), backing_queue :: atom(), backing_queue_state :: any(), - seen_status :: ?DICT_TYPE(), + seen_status :: dict:dict(), confirmed :: [rabbit_guid:guid()], - known_senders :: ?SET_TYPE() + known_senders :: sets:set() }. -spec promote_backing_queue_state (rabbit_amqqueue:name(), pid(), atom(), any(), pid(), [any()], - ?DICT_TYPE(), [pid()]) -> + dict:dict(), [pid()]) -> master_state(). -spec sender_death_fun() -> death_fun(). @@ -219,16 +219,24 @@ stop_all_slaves(Reason, #state{name = QName, gm = GM, wait_timeout = WT}) -> %% monitor them but they would not have received the GM %% message. So only wait for slaves which are still %% not-partitioned. - [receive - {'DOWN', MRef, process, _Pid, _Info} -> - ok - after WT -> - rabbit_mirror_queue_misc:log_warning( - QName, "Missing 'DOWN' message from ~p in node ~p~n", - [Pid, node(Pid)]), - ok - end - || {Pid, MRef} <- PidsMRefs, rabbit_mnesia:on_running_node(Pid)], + PendingSlavePids = + lists:foldl( + fun({Pid, MRef}, Acc) -> + case rabbit_mnesia:on_running_node(Pid) of + true -> + receive + {'DOWN', MRef, process, _Pid, _Info} -> + Acc + after WT -> + rabbit_mirror_queue_misc:log_warning( + QName, "Missing 'DOWN' message from ~p in" + " node ~p~n", [Pid, node(Pid)]), + [Pid | Acc] + end; + false -> + Acc + end + end, [], PidsMRefs), %% Normally when we remove a slave another slave or master will %% notice and update Mnesia. But we just removed them all, and %% have stopped listening ourselves. So manually clean up. @@ -236,7 +244,11 @@ stop_all_slaves(Reason, #state{name = QName, gm = GM, wait_timeout = WT}) -> fun () -> [Q] = mnesia:read({rabbit_queue, QName}), rabbit_mirror_queue_misc:store_updated_slaves( - Q #amqqueue { gm_pids = [], slave_pids = [] }) + Q #amqqueue { gm_pids = [], slave_pids = [], + %% Restarted slaves on running nodes can + %% ensure old incarnations are stopped using + %% the pending slave pids. + slave_pids_pending_shutdown = PendingSlavePids}) end), ok = gm:forget_group(QName). diff --git a/src/rabbit_mirror_queue_misc.erl b/src/rabbit_mirror_queue_misc.erl index 375a0366dd..59522da4a9 100644 --- a/src/rabbit_mirror_queue_misc.erl +++ b/src/rabbit_mirror_queue_misc.erl @@ -64,6 +64,8 @@ -spec is_mirrored(rabbit_types:amqqueue()) -> boolean(). -spec update_mirrors (rabbit_types:amqqueue(), rabbit_types:amqqueue()) -> 'ok'. +-spec update_mirrors + (rabbit_types:amqqueue()) -> 'ok'. -spec maybe_drop_master_after_sync(rabbit_types:amqqueue()) -> 'ok'. -spec maybe_auto_sync(rabbit_types:amqqueue()) -> 'ok'. -spec log_info(rabbit_amqqueue:name(), string(), [any()]) -> 'ok'. diff --git a/src/rabbit_mirror_queue_mode.erl b/src/rabbit_mirror_queue_mode.erl index 3733c7f0f8..47b150b5a0 100644 --- a/src/rabbit_mirror_queue_mode.erl +++ b/src/rabbit_mirror_queue_mode.erl @@ -16,6 +16,10 @@ -module(rabbit_mirror_queue_mode). +-behaviour(rabbit_registry_class). + +-export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). + -type master() :: node(). -type slave() :: node(). -type params() :: any(). @@ -42,3 +46,6 @@ %% Are the parameters valid for this mode? -callback validate_policy(params()) -> rabbit_policy_validator:validate_results(). + +added_to_rabbit_registry(_Type, _ModuleName) -> ok. +removed_from_rabbit_registry(_Type) -> ok. diff --git a/src/rabbit_mirror_queue_mode_exactly.erl b/src/rabbit_mirror_queue_mode_exactly.erl index 593f0a4138..c2ffa39f59 100644 --- a/src/rabbit_mirror_queue_mode_exactly.erl +++ b/src/rabbit_mirror_queue_mode_exactly.erl @@ -45,7 +45,7 @@ suggested_queue_nodes(Count, MNode, SNodes, _SSNodes, Poss) -> end}. shuffle(L) -> - {_, L1} = lists:unzip(lists:keysort(1, [{rand_compat:uniform(), N} || N <- L])), + {_, L1} = lists:unzip(lists:keysort(1, [{rand:uniform(), N} || N <- L])), L1. validate_policy(N) when is_integer(N) andalso N > 0 -> diff --git a/src/rabbit_mirror_queue_slave.erl b/src/rabbit_mirror_queue_slave.erl index 6017e5a028..61623c9441 100644 --- a/src/rabbit_mirror_queue_slave.erl +++ b/src/rabbit_mirror_queue_slave.erl @@ -163,9 +163,11 @@ handle_go(Q = #amqqueue{name = QName}) -> init_it(Self, GM, Node, QName) -> case mnesia:read({rabbit_queue, QName}) of - [Q = #amqqueue { pid = QPid, slave_pids = SPids, gm_pids = GMPids }] -> + [Q = #amqqueue { pid = QPid, slave_pids = SPids, gm_pids = GMPids, + slave_pids_pending_shutdown = PSPids}] -> case [Pid || Pid <- [QPid | SPids], node(Pid) =:= Node] of - [] -> add_slave(Q, Self, GM), + [] -> stop_pending_slaves(QName, PSPids), + add_slave(Q, Self, GM), {new, QPid, GMPids}; [QPid] -> case rabbit_mnesia:is_process_alive(QPid) of true -> duplicate_live_master; @@ -186,6 +188,26 @@ init_it(Self, GM, Node, QName) -> master_in_recovery end. +%% Pending slaves have been asked to stop by the master, but despite the node +%% being up these did not answer on the expected timeout. Stop local slaves now. +stop_pending_slaves(QName, Pids) -> + [begin + rabbit_mirror_queue_misc:log_warning( + QName, "Detected stale HA slave, stopping it: ~p~n", [Pid]), + case erlang:process_info(Pid, dictionary) of + undefined -> ok; + {dictionary, Dict} -> + case proplists:get_value('$ancestors', Dict) of + [Sup, rabbit_amqqueue_sup_sup | _] -> + exit(Sup, kill), + exit(Pid, kill); + _ -> + ok + end + end + end || Pid <- Pids, node(Pid) =:= node(), + true =:= erlang:is_process_alive(Pid)]. + %% Add to the end, so they are in descending order of age, see %% rabbit_mirror_queue_misc:promote_slave/1 add_slave(Q = #amqqueue { slave_pids = SPids, gm_pids = GMPids }, New, GM) -> diff --git a/src/rabbit_mirror_queue_sync.erl b/src/rabbit_mirror_queue_sync.erl index c438e91a3f..d6357e0dc0 100644 --- a/src/rabbit_mirror_queue_sync.erl +++ b/src/rabbit_mirror_queue_sync.erl @@ -104,7 +104,7 @@ master_batch_go0(Args, BatchSize, BQ, BQS) -> false -> {cont, Acc1} end end, - FoldAcc = {[], 0, {0, BQ:depth(BQS)}, time_compat:monotonic_time()}, + FoldAcc = {[], 0, {0, BQ:depth(BQS)}, erlang:monotonic_time()}, bq_fold(FoldFun, FoldAcc, Args, BQ, BQS). master_batch_send({Syncer, Ref, Log, HandleInfo, EmitStats, Parent}, @@ -164,12 +164,12 @@ stop_syncer(Syncer, Msg) -> end. maybe_emit_stats(Last, I, EmitStats, Log) -> - Interval = time_compat:convert_time_unit( - time_compat:monotonic_time() - Last, native, micro_seconds), + Interval = erlang:convert_time_unit( + erlang:monotonic_time() - Last, native, micro_seconds), case Interval > ?SYNC_PROGRESS_INTERVAL of true -> EmitStats({syncing, I}), Log("~p messages", [I]), - time_compat:monotonic_time(); + erlang:monotonic_time(); false -> Last end. diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 596eb62b03..d66f5b8fab 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -98,8 +98,11 @@ init() -> ensure_mnesia_dir(), case is_virgin_node() of true -> - rabbit_log:info("Database directory at ~s is empty. Initialising from scratch...~n", + rabbit_log:info("Database directory at ~s is empty. " + "Assuming we need to join an existing cluster or initialise from scratch...~n", [dir()]), + rabbit_log:info("Using ~p as peer discovery backend~n", + [rabbit_peer_discovery:backend()]), init_from_config(); false -> NodeType = node_type(), @@ -117,10 +120,10 @@ init_from_config() -> (Name, BadNames) when is_atom(Name) -> BadNames; (Name, BadNames) -> [Name | BadNames] end, - {TryNodes, NodeType} = - case application:get_env(rabbit, cluster_nodes) of + {DiscoveredNodes, NodeType} = + case rabbit_peer_discovery:discover_cluster_nodes() of {ok, {Nodes, Type} = Config} - when is_list(Nodes) andalso (Type == disc orelse Type == ram) -> + when is_list(Nodes) andalso (Type == disc orelse Type == disk orelse Type == ram) -> case lists:foldr(FindBadNodeNames, [], Nodes) of [] -> Config; BadNames -> e({invalid_cluster_node_names, BadNames}) @@ -137,9 +140,12 @@ init_from_config() -> {ok, _} -> e(invalid_cluster_nodes_conf) end, - case TryNodes of + case DiscoveredNodes of [] -> init_db_and_upgrade([node()], disc, false); - _ -> auto_cluster(TryNodes, NodeType) + _ -> + rabbit_log:info("Discovered peer nodes: ~s~n", + [rabbit_peer_discovery:format_discovered_nodes(DiscoveredNodes)]), + auto_cluster(DiscoveredNodes, NodeType) end. auto_cluster(TryNodes, NodeType) -> @@ -148,6 +154,7 @@ auto_cluster(TryNodes, NodeType) -> rabbit_log:info("Node '~p' selected for auto-clustering~n", [Node]), {ok, {_, DiscNodes, _}} = discover_cluster0(Node), init_db_and_upgrade(DiscNodes, NodeType, true), + rabbit_connection_tracking:boot(), rabbit_node_monitor:notify_joined_cluster(); none -> rabbit_log:warning( @@ -194,6 +201,7 @@ join_cluster(DiscoveryNode, NodeType) -> [ClusterNodes, NodeType]), ok = init_db_with_mnesia(ClusterNodes, NodeType, true, true), + rabbit_connection_tracking:boot(), rabbit_node_monitor:notify_joined_cluster(), ok; {error, Reason} -> @@ -295,6 +303,9 @@ update_cluster_nodes(DiscoveryNode) -> %% the last or second to last after the node we're removing to go %% down forget_cluster_node(Node, RemoveWhenOffline) -> + forget_cluster_node(Node, RemoveWhenOffline, true). + +forget_cluster_node(Node, RemoveWhenOffline, EmitNodeDeletedEvent) -> case lists:member(Node, cluster_nodes(all)) of true -> ok; false -> e(not_a_cluster_node) @@ -306,6 +317,9 @@ forget_cluster_node(Node, RemoveWhenOffline) -> {false, true} -> rabbit_log:info( "Removing node ~p from cluster~n", [Node]), case remove_node_if_mnesia_running(Node) of + ok when EmitNodeDeletedEvent -> + rabbit_event:notify(node_deleted, [{node, Node}]), + ok; ok -> ok; {error, _} = Err -> throw(Err) end @@ -326,7 +340,10 @@ remove_node_offline_node(Node) -> %% they are loaded. rabbit_table:force_load(), rabbit_table:wait_for_replicated(), - forget_cluster_node(Node, false), + %% We skip the 'node_deleted' event because the + %% application is stopped and thus, rabbit_event is not + %% enabled. + forget_cluster_node(Node, false, false), force_load_next_boot() after stop_mnesia() @@ -880,7 +897,7 @@ find_auto_cluster_node([]) -> find_auto_cluster_node([Node | Nodes]) -> Fail = fun (Fmt, Args) -> rabbit_log:warning( - "Could not auto-cluster with ~s: " ++ Fmt, [Node | Args]), + "Could not auto-cluster with node ~s: " ++ Fmt, [Node | Args]), find_auto_cluster_node(Nodes) end, case remote_node_info(Node) of diff --git a/src/rabbit_mnesia_rename.erl b/src/rabbit_mnesia_rename.erl index 0945e31522..0c3e7c2366 100644 --- a/src/rabbit_mnesia_rename.erl +++ b/src/rabbit_mnesia_rename.erl @@ -124,7 +124,13 @@ prepare(Node, NodeMapList) -> take_backup(Backup) -> start_mnesia(), - ok = mnesia:backup(Backup), + %% We backup only local tables: in particular, this excludes the + %% connection tracking tables which have no local replica. + LocalTables = mnesia:system_info(local_tables), + {ok, Name, _Nodes} = mnesia:activate_checkpoint([ + {max, LocalTables} + ]), + ok = mnesia:backup_checkpoint(Name, Backup), stop_mnesia(). restore_backup(Backup) -> diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index d3ff077c8b..8e2b1c0d49 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -157,7 +157,7 @@ -type client_msstate() :: #client_msstate { server :: server(), client_ref :: client_ref(), - file_handle_cache :: ?DICT_TYPE(), + file_handle_cache :: dict:dict(), index_state :: any(), index_module :: atom(), dir :: file:filename(), @@ -171,7 +171,7 @@ fun ((A) -> 'finished' | {rabbit_types:msg_id(), non_neg_integer(), A}). -type maybe_msg_id_fun() :: - 'undefined' | fun ((?GB_SET_TYPE(), 'written' | 'ignored') -> any()). + 'undefined' | fun ((gb_sets:set(), 'written' | 'ignored') -> any()). -type maybe_close_fds_fun() :: 'undefined' | fun (() -> 'ok'). -type deletion_thunk() :: fun (() -> boolean()). diff --git a/src/rabbit_parameter_validation.erl b/src/rabbit_parameter_validation.erl index 90ab1d5286..00e83d2757 100644 --- a/src/rabbit_parameter_validation.erl +++ b/src/rabbit_parameter_validation.erl @@ -16,7 +16,7 @@ -module(rabbit_parameter_validation). --export([number/2, binary/2, boolean/2, list/2, regex/2, proplist/3, enum/1]). +-export([number/2, integer/2, binary/2, boolean/2, list/2, regex/2, proplist/3, enum/1]). number(_Name, Term) when is_number(Term) -> ok; @@ -24,6 +24,12 @@ number(_Name, Term) when is_number(Term) -> number(Name, Term) -> {error, "~s should be number, actually was ~p", [Name, Term]}. +integer(_Name, Term) when is_integer(Term) -> + ok; + +integer(Name, Term) -> + {error, "~s should be number, actually was ~p", [Name, Term]}. + binary(_Name, Term) when is_binary(Term) -> ok; diff --git a/src/rabbit_password.erl b/src/rabbit_password.erl index b7987df1d8..0538445ab4 100644 --- a/src/rabbit_password.erl +++ b/src/rabbit_password.erl @@ -35,7 +35,7 @@ hash(HashingMod, Cleartext) -> <<SaltBin/binary, Hash/binary>>. generate_salt() -> - Salt = rand_compat:uniform(16#ffffffff), + Salt = rand:uniform(16#ffffffff), <<Salt:32>>. salted_hash(Salt, Cleartext) -> diff --git a/src/rabbit_peer_discovery.erl b/src/rabbit_peer_discovery.erl new file mode 100644 index 0000000000..965be3946d --- /dev/null +++ b/src/rabbit_peer_discovery.erl @@ -0,0 +1,59 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is Pivotal Software, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_peer_discovery). + +%% API +-export([discover_cluster_nodes/0, backend/0, + normalize/1, format_discovered_nodes/1]). + + + +-spec backend() -> atom(). + +backend() -> + case application:get_env(rabbit, peer_discovery_backend) of + {ok, Backend} when is_atom(Backend) -> Backend; + undefined -> rabbit_peer_discovery_classic_config + end. + + +-spec discover_cluster_nodes() -> {ok, Nodes :: list()} | + {ok, {Nodes :: list(), NodeType :: rabbit_types:node_type()}} | + {error, Reason :: string()}. + +discover_cluster_nodes() -> + Backend = backend(), + normalize(Backend:list_nodes()). + + +-spec normalize({ok, Nodes :: list()} | + {ok, {Nodes :: list(), NodeType :: rabbit_types:node_type()}} | + {error, Reason :: string()}) -> {ok, {Nodes :: list(), NodeType :: rabbit_types:node_type()}} | + {error, Reason :: string()}. + +normalize({ok, Nodes}) when is_list(Nodes) -> + {ok, {Nodes, disc}}; +normalize({ok, {Nodes, NodeType}}) when is_list(Nodes) andalso is_atom(NodeType) -> + {ok, {Nodes, NodeType}}; +normalize({error, Reason}) -> + {error, Reason}. + + +-spec format_discovered_nodes(Nodes :: list()) -> string(). + +format_discovered_nodes(Nodes) -> + string:join(lists:map(fun (Val) -> hd(io_lib:format("~s", [Val])) end, Nodes), ", "). diff --git a/src/rabbit_peer_discovery_classic_config.erl b/src/rabbit_peer_discovery_classic_config.erl new file mode 100644 index 0000000000..a3d3d50617 --- /dev/null +++ b/src/rabbit_peer_discovery_classic_config.erl @@ -0,0 +1,45 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_peer_discovery_classic_config). +-behaviour(rabbit_peer_discovery_backend). + +-include("rabbit.hrl"). + +-export([list_nodes/0, register/0, unregister/0]). + +%% +%% API +%% + +-spec list_nodes() -> {ok, Nodes :: list()} | {error, Reason :: string()}. + +list_nodes() -> + case application:get_env(rabbit, cluster_nodes) of + {_Nodes, _NodeType} = Pair -> Pair; + Nodes when is_list(Nodes) -> {Nodes, disc}; + undefined -> {[], disc} + end. + +-spec register() -> ok. + +register() -> + ok. + +-spec unregister() -> ok. + +unregister() -> + ok. diff --git a/src/rabbit_plugins.erl b/src/rabbit_plugins.erl index 4d8966f7e2..723f725303 100644 --- a/src/rabbit_plugins.erl +++ b/src/rabbit_plugins.erl @@ -16,10 +16,15 @@ -module(rabbit_plugins). -include("rabbit.hrl"). +-include_lib("stdlib/include/zip.hrl"). -export([setup/0, active/0, read_enabled/1, list/1, list/2, dependencies/3]). -export([ensure/1]). +-export([extract_schemas/1]). +-export([validate_plugins/1, format_invalid_plugins/1]). +% Export for testing purpose. +-export([is_version_supported/2, validate_plugins/2]). %%---------------------------------------------------------------------------- -type plugin_name() :: atom(). @@ -43,6 +48,7 @@ ensure(FileJustChanged0) -> FileJustChanged -> Enabled = read_enabled(OurFile), Wanted = prepare_plugins(Enabled), + rabbit_config:prepare_and_use_config(), Current = active(), Start = Wanted -- Current, Stop = Current -- Wanted, @@ -76,6 +82,50 @@ setup() -> Enabled = read_enabled(EnabledFile), prepare_plugins(Enabled). +extract_schemas(SchemaDir) -> + application:load(rabbit), + {ok, EnabledFile} = application:get_env(rabbit, enabled_plugins_file), + Enabled = read_enabled(EnabledFile), + + {ok, PluginsDistDir} = application:get_env(rabbit, plugins_dir), + + AllPlugins = list(PluginsDistDir), + Wanted = dependencies(false, Enabled, AllPlugins), + WantedPlugins = lookup_plugins(Wanted, AllPlugins), + [ extract_schema(Plugin, SchemaDir) || Plugin <- WantedPlugins ], + application:unload(rabbit), + ok. + +extract_schema(#plugin{type = ez, location = Location}, SchemaDir) -> + {ok, Files} = zip:extract(Location, + [memory, {file_filter, + fun(#zip_file{name = Name}) -> + string:str(Name, "priv/schema") > 0 + end}]), + lists:foreach( + fun({FileName, Content}) -> + ok = file:write_file(filename:join([SchemaDir, + filename:basename(FileName)]), + Content) + end, + Files), + ok; +extract_schema(#plugin{type = dir, location = Location}, SchemaDir) -> + PluginSchema = filename:join([Location, + "priv", + "schema"]), + case rabbit_file:is_dir(PluginSchema) of + false -> ok; + true -> + PluginSchemaFiles = + [ filename:join(PluginSchema, FileName) + || FileName <- rabbit_file:wildcard(".*\\.schema", + PluginSchema) ], + [ file:copy(SchemaFile, SchemaDir) + || SchemaFile <- PluginSchemaFiles ] + end. + + %% @doc Lists the plugins which are currently running. active() -> {ok, ExpandDir} = application:get_env(rabbit, plugins_expand_dir), @@ -96,27 +146,31 @@ list(PluginsDir, IncludeRequiredDeps) -> %% instance. application:load(rabbit), {ok, RabbitDeps} = application:get_key(rabbit, applications), + AllPlugins = [plugin_info(PluginsDir, Plug) || Plug <- EZs ++ FreeApps], {AvailablePlugins, Problems} = - lists:foldl(fun ({error, EZ, Reason}, {Plugins1, Problems1}) -> - {Plugins1, [{EZ, Reason} | Problems1]}; - (Plugin = #plugin{name = Name}, {Plugins1, Problems1}) -> - %% Applications RabbitMQ depends on (eg. - %% "rabbit_common") can't be considered - %% plugins, otherwise rabbitmq-plugins would - %% list them and the user may believe he can - %% disable them. - case IncludeRequiredDeps orelse - not lists:member(Name, RabbitDeps) of - true -> {[Plugin|Plugins1], Problems1}; - false -> {Plugins1, Problems1} - end - end, {[], []}, - [plugin_info(PluginsDir, Plug) || Plug <- EZs ++ FreeApps]), + lists:foldl( + fun ({error, EZ, Reason}, {Plugins1, Problems1}) -> + {Plugins1, [{EZ, Reason} | Problems1]}; + (Plugin = #plugin{name = Name}, + {Plugins1, Problems1}) -> + %% Applications RabbitMQ depends on (eg. + %% "rabbit_common") can't be considered + %% plugins, otherwise rabbitmq-plugins would + %% list them and the user may believe he can + %% disable them. + case IncludeRequiredDeps orelse + not lists:member(Name, RabbitDeps) of + true -> {[Plugin|Plugins1], Problems1}; + false -> {Plugins1, Problems1} + end + end, {[], []}, + AllPlugins), case Problems of [] -> ok; _ -> rabbit_log:warning( "Problem reading some plugins: ~p~n", [Problems]) end, + Plugins = lists:filter(fun(P) -> not plugin_provided_by_otp(P) end, AvailablePlugins), ensure_dependencies(Plugins). @@ -146,8 +200,9 @@ dependencies(Reverse, Sources, AllPlugins) -> false -> digraph_utils:reachable(Sources, G); true -> digraph_utils:reaching(Sources, G) end, + OrderedDests = digraph_utils:postorder(digraph_utils:subgraph(G, Dests)), true = digraph:delete(G), - Dests. + OrderedDests. %% For a few known cases, an externally provided plugin can be trusted. %% In this special case, it overrides the plugin. @@ -195,19 +250,130 @@ prepare_plugins(Enabled) -> AllPlugins = list(PluginsDistDir), Wanted = dependencies(false, Enabled, AllPlugins), WantedPlugins = lookup_plugins(Wanted, AllPlugins), - + {ValidPlugins, Problems} = validate_plugins(WantedPlugins), + maybe_warn_about_invalid_plugins(Problems), case filelib:ensure_dir(ExpandDir ++ "/") of ok -> ok; {error, E2} -> throw({error, {cannot_create_plugins_expand_dir, [ExpandDir, E2]}}) end, - - [prepare_plugin(Plugin, ExpandDir) || Plugin <- WantedPlugins], + [prepare_plugin(Plugin, ExpandDir) || Plugin <- ValidPlugins], [prepare_dir_plugin(PluginAppDescPath) || PluginAppDescPath <- filelib:wildcard(ExpandDir ++ "/*/ebin/*.app")], Wanted. +maybe_warn_about_invalid_plugins([]) -> + ok; +maybe_warn_about_invalid_plugins(InvalidPlugins) -> + %% TODO: error message formatting + rabbit_log:warning(format_invalid_plugins(InvalidPlugins)). + + +format_invalid_plugins(InvalidPlugins) -> + lists:flatten(["Failed to enable some plugins: \r\n" + | [format_invalid_plugin(Plugin) + || Plugin <- InvalidPlugins]]). + +format_invalid_plugin({Name, Errors}) -> + [io_lib:format(" ~p:~n", [Name]) + | [format_invalid_plugin_error(Err) || Err <- Errors]]. + +format_invalid_plugin_error({missing_dependency, Dep}) -> + io_lib:format(" Dependency is missing or invalid: ~p~n", [Dep]); +%% a plugin doesn't support the effective broker version +format_invalid_plugin_error({broker_version_mismatch, Version, Required}) -> + io_lib:format(" Plugin doesn't support current server version." + " Actual broker version: ~p, supported by the plugin: ~p~n", [Version, Required]); +%% one of dependencies of a plugin doesn't match its version requirements +format_invalid_plugin_error({{dependency_version_mismatch, Version, Required}, Name}) -> + io_lib:format(" Version '~p' of dependency '~p' is unsupported." + " Version ranges supported by the plugin: ~p~n", + [Version, Name, Required]); +format_invalid_plugin_error(Err) -> + io_lib:format(" Unknown error ~p~n", [Err]). + +validate_plugins(Plugins) -> + application:load(rabbit), + RabbitVersion = RabbitVersion = case application:get_key(rabbit, vsn) of + undefined -> "0.0.0"; + {ok, Val} -> Val + end, + validate_plugins(Plugins, RabbitVersion). + +validate_plugins(Plugins, BrokerVersion) -> + lists:foldl( + fun(#plugin{name = Name, + broker_version_requirements = BrokerVersionReqs, + dependency_version_requirements = DepsVersions} = Plugin, + {Plugins0, Errors}) -> + case is_version_supported(BrokerVersion, BrokerVersionReqs) of + true -> + case BrokerVersion of + "0.0.0" -> + rabbit_log:warning( + "Running development version of the broker." + " Requirement ~p for plugin ~p is ignored.", + [BrokerVersionReqs, Name]); + _ -> ok + end, + case check_plugins_versions(Name, Plugins0, DepsVersions) of + ok -> {[Plugin | Plugins0], Errors}; + {error, Err} -> {Plugins0, [{Name, Err} | Errors]} + end; + false -> + Error = [{broker_version_mismatch, BrokerVersion, BrokerVersionReqs}], + {Plugins0, [{Name, Error} | Errors]} + end + end, + {[],[]}, + Plugins). + +check_plugins_versions(PluginName, AllPlugins, RequiredVersions) -> + ExistingVersions = [{Name, Vsn} + || #plugin{name = Name, version = Vsn} <- AllPlugins], + Problems = lists:foldl( + fun({Name, Versions}, Acc) -> + case proplists:get_value(Name, ExistingVersions) of + undefined -> [{missing_dependency, Name} | Acc]; + Version -> + case is_version_supported(Version, Versions) of + true -> + case Version of + "" -> + rabbit_log:warning( + "~p plugin version is not defined." + " Requirement ~p for plugin ~p is ignored", + [Versions, PluginName]); + _ -> ok + end, + Acc; + false -> + [{{dependency_version_mismatch, Version, Versions}, Name} | Acc] + end + end + end, + [], + RequiredVersions), + case Problems of + [] -> ok; + _ -> {error, Problems} + end. + +is_version_supported("", _) -> true; +is_version_supported("0.0.0", _) -> true; +is_version_supported(_Version, []) -> true; +is_version_supported(Version, ExpectedVersions) -> + case lists:any(fun(ExpectedVersion) -> + rabbit_misc:version_minor_equivalent(ExpectedVersion, Version) + andalso + rabbit_misc:version_compare(ExpectedVersion, Version, lte) + end, + ExpectedVersions) of + true -> true; + false -> false + end. + clean_plugins(Plugins) -> {ok, ExpandDir} = application:get_env(rabbit, plugins_expand_dir), [clean_plugin(Plugin, ExpandDir) || Plugin <- Plugins]. @@ -280,8 +446,12 @@ mkplugin(Name, Props, Type, Location) -> Version = proplists:get_value(vsn, Props, "0"), Description = proplists:get_value(description, Props, ""), Dependencies = proplists:get_value(applications, Props, []), + BrokerVersions = proplists:get_value(broker_version_requirements, Props, []), + DepsVersions = proplists:get_value(dependency_version_requirements, Props, []), #plugin{name = Name, version = Version, description = Description, - dependencies = Dependencies, location = Location, type = Type}. + dependencies = Dependencies, location = Location, type = Type, + broker_version_requirements = BrokerVersions, + dependency_version_requirements = DepsVersions}. read_app_file(EZ) -> case zip:list_dir(EZ) of @@ -316,4 +486,9 @@ plugin_names(Plugins) -> [Name || #plugin{name = Name} <- Plugins]. lookup_plugins(Names, AllPlugins) -> - [P || P = #plugin{name = Name} <- AllPlugins, lists:member(Name, Names)]. + % Preserve order of Names + lists:map( + fun(Name) -> + lists:keyfind(Name, #plugin.name, AllPlugins) + end, + Names). diff --git a/src/rabbit_plugins_main.erl b/src/rabbit_plugins_main.erl index ff516268c6..4618249ee8 100644 --- a/src/rabbit_plugins_main.erl +++ b/src/rabbit_plugins_main.erl @@ -95,6 +95,12 @@ action(enable, Node, ToEnable0, Opts, State = #cli{all = All, _ -> throw({error_string, fmt_missing(Missing)}) end, NewEnabled = lists:usort(Enabled ++ ToEnable), + Invalid = validate_plugins(NewEnabled, State), + case Invalid of + [] -> ok; + _ -> throw({error_string, + rabbit_plugins:format_invalid_plugins(Invalid)}) + end, NewImplicit = write_enabled_plugins(NewEnabled, State), case NewEnabled -- Implicit of [] -> io:format("Plugin configuration unchanged.~n"); @@ -111,6 +117,12 @@ action(set, Node, NewEnabled0, Opts, State = #cli{all = All, [] -> ok; _ -> throw({error_string, fmt_missing(Missing)}) end, + Invalid = validate_plugins(NewEnabled, State), + case Invalid of + [] -> ok; + _ -> throw({error_string, + rabbit_plugins:format_invalid_plugins(Invalid)}) + end, NewImplicit = write_enabled_plugins(NewEnabled, State), case NewImplicit of [] -> io:format("All plugins are now disabled.~n"); @@ -151,6 +163,16 @@ action(help, _Node, _Args, _Opts, _State) -> %%---------------------------------------------------------------------------- +validate_plugins(Names, #cli{all = All}) -> + Deps = rabbit_plugins:dependencies(false, Names, All), + DepsPlugins = lists:map( + fun(Name) -> + lists:keyfind(Name, #plugin.name, All) + end, + Deps), + {_, Errors} = rabbit_plugins:validate_plugins(DepsPlugins), + Errors. + %% Pretty print a list of plugins. format_plugins(Node, Pattern, Opts, #cli{all = All, enabled = Enabled, diff --git a/src/rabbit_policies.erl b/src/rabbit_policies.erl index c7d4c99f37..22eb3fde18 100644 --- a/src/rabbit_policies.erl +++ b/src/rabbit_policies.erl @@ -20,10 +20,11 @@ %% validation functions. -behaviour(rabbit_policy_validator). +-behaviour(rabbit_policy_merge_strategy). -include("rabbit.hrl"). --export([register/0, validate_policy/1]). +-export([register/0, validate_policy/1, merge_policy_value/3]). -rabbit_boot_step({?MODULE, [{description, "internal policies"}, @@ -40,7 +41,15 @@ register() -> {policy_validator, <<"expires">>}, {policy_validator, <<"max-length">>}, {policy_validator, <<"max-length-bytes">>}, - {policy_validator, <<"queue-mode">>}]], + {policy_validator, <<"queue-mode">>}, + {operator_policy_validator, <<"expires">>}, + {operator_policy_validator, <<"message-ttl">>}, + {operator_policy_validator, <<"max-length">>}, + {operator_policy_validator, <<"max-length-bytes">>}, + {policy_merge_strategy, <<"expires">>}, + {policy_merge_strategy, <<"message-ttl">>}, + {policy_merge_strategy, <<"max-length">>}, + {policy_merge_strategy, <<"max-length-bytes">>}]], ok. validate_policy(Terms) -> @@ -96,3 +105,9 @@ validate_policy0(<<"queue-mode">>, <<"lazy">>) -> ok; validate_policy0(<<"queue-mode">>, Value) -> {error, "~p is not a valid queue-mode value", [Value]}. + +merge_policy_value(<<"message-ttl">>, Val, OpVal) -> min(Val, OpVal); +merge_policy_value(<<"max-length">>, Val, OpVal) -> min(Val, OpVal); +merge_policy_value(<<"max-length-bytes">>, Val, OpVal) -> min(Val, OpVal); +merge_policy_value(<<"expires">>, Val, OpVal) -> min(Val, OpVal). + diff --git a/src/rabbit_policy.erl b/src/rabbit_policy.erl index a9caadf972..7e39164882 100644 --- a/src/rabbit_policy.erl +++ b/src/rabbit_policy.erl @@ -38,14 +38,16 @@ -include("rabbit.hrl"). --import(rabbit_misc, [pget/2]). +-import(rabbit_misc, [pget/2, pget/3]). -export([register/0]). -export([invalidate/0, recover/0]). --export([name/1, get/2, get_arg/3, set/1]). +-export([name/1, name_op/1, effective_definition/1, get/2, get_arg/3, set/1]). -export([validate/5, notify/4, notify_clear/3]). -export([parse_set/6, set/6, delete/2, lookup/2, list/0, list/1, list_formatted/1, list_formatted/3, info_keys/0]). +-export([parse_set_op/6, set_op/6, delete_op/2, lookup_op/2, list_op/0, list_op/1, + list_formatted_op/1, list_formatted_op/3]). -rabbit_boot_step({?MODULE, [{description, "policy parameters"}, @@ -54,30 +56,88 @@ {enables, recovery}]}). register() -> - rabbit_registry:register(runtime_parameter, <<"policy">>, ?MODULE). + rabbit_registry:register(runtime_parameter, <<"policy">>, ?MODULE), + rabbit_registry:register(runtime_parameter, <<"operator_policy">>, ?MODULE). name(#amqqueue{policy = Policy}) -> name0(Policy); name(#exchange{policy = Policy}) -> name0(Policy). +name_op(#amqqueue{operator_policy = Policy}) -> name0(Policy); +name_op(#exchange{operator_policy = Policy}) -> name0(Policy). + name0(undefined) -> none; name0(Policy) -> pget(name, Policy). -set(Q = #amqqueue{name = Name}) -> Q#amqqueue{policy = set0(Name)}; -set(X = #exchange{name = Name}) -> X#exchange{policy = set0(Name)}. - -set0(Name = #resource{virtual_host = VHost}) -> match(Name, list(VHost)). +effective_definition(#amqqueue{policy = Policy, operator_policy = OpPolicy}) -> + effective_definition0(Policy, OpPolicy); +effective_definition(#exchange{policy = Policy, operator_policy = OpPolicy}) -> + effective_definition0(Policy, OpPolicy). + +effective_definition0(undefined, undefined) -> undefined; +effective_definition0(Policy, undefined) -> pget(definition, Policy); +effective_definition0(undefined, OpPolicy) -> pget(definition, OpPolicy); +effective_definition0(Policy, OpPolicy) -> + OpDefinition = pget(definition, OpPolicy, []), + Definition = pget(definition, Policy, []), + {Keys, _} = lists:unzip(Definition), + {OpKeys, _} = lists:unzip(OpDefinition), + lists:map(fun(Key) -> + case {pget(Key, Definition), pget(Key, OpDefinition)} of + {Val, undefined} -> {Key, Val}; + {undefined, Val} -> {Key, Val}; + {Val, OpVal} -> {Key, merge_policy_value(Key, Val, OpVal)} + end + end, + lists:umerge(Keys, OpKeys)). + +set(Q = #amqqueue{name = Name}) -> Q#amqqueue{policy = match(Name), + operator_policy = match_op(Name)}; +set(X = #exchange{name = Name}) -> X#exchange{policy = match(Name), + operator_policy = match_op(Name)}. + +match(Name = #resource{virtual_host = VHost}) -> + match(Name, list(VHost)). + +match_op(Name = #resource{virtual_host = VHost}) -> + match(Name, list_op(VHost)). + +get(Name, #amqqueue{policy = Policy, operator_policy = OpPolicy}) -> + get0(Name, Policy, OpPolicy); +get(Name, #exchange{policy = Policy, operator_policy = OpPolicy}) -> + get0(Name, Policy, OpPolicy); -get(Name, #amqqueue{policy = Policy}) -> get0(Name, Policy); -get(Name, #exchange{policy = Policy}) -> get0(Name, Policy); %% Caution - SLOW. get(Name, EntityName = #resource{virtual_host = VHost}) -> - get0(Name, match(EntityName, list(VHost))). + get0(Name, + match(EntityName, list(VHost)), + match(EntityName, list_op(VHost))). + +get0(_Name, undefined, undefined) -> undefined; +get0(Name, undefined, OpPolicy) -> pget(Name, pget(definition, OpPolicy, [])); +get0(Name, Policy, undefined) -> pget(Name, pget(definition, Policy, [])); +get0(Name, Policy, OpPolicy) -> + OpDefinition = pget(definition, OpPolicy, []), + Definition = pget(definition, Policy, []), + case {pget(Name, Definition), pget(Name, OpDefinition)} of + {undefined, undefined} -> undefined; + {Val, undefined} -> Val; + {undefined, Val} -> Val; + {Val, OpVal} -> merge_policy_value(Name, Val, OpVal) + end. + +merge_policy_value(Name, PolicyVal, OpVal) -> + case policy_merge_strategy(Name) of + {ok, Module} -> Module:merge_policy_value(Name, PolicyVal, OpVal); + {error, not_found} -> PolicyVal + end. -get0(_Name, undefined) -> undefined; -get0(Name, List) -> case pget(definition, List) of - undefined -> undefined; - Policy -> pget(Name, Policy) - end. +policy_merge_strategy(Name) -> + case rabbit_registry:binary_to_type(Name) of + {error, not_found} -> + {error, not_found}; + T -> + rabbit_registry:lookup_module(policy_merge_strategy, T) + end. %% Many heads for optimisation get_arg(_AName, _PName, #exchange{arguments = [], policy = undefined}) -> @@ -112,19 +172,24 @@ recover0() -> Xs = mnesia:dirty_match_object(rabbit_durable_exchange, #exchange{_ = '_'}), Qs = mnesia:dirty_match_object(rabbit_durable_queue, #amqqueue{_ = '_'}), Policies = list(), + OpPolicies = list_op(), [rabbit_misc:execute_mnesia_transaction( fun () -> mnesia:write( rabbit_durable_exchange, rabbit_exchange_decorator:set( - X#exchange{policy = match(Name, Policies)}), write) + X#exchange{policy = match(Name, Policies), + operator_policy = match(Name, OpPolicies)}), + write) end) || X = #exchange{name = Name} <- Xs], [rabbit_misc:execute_mnesia_transaction( fun () -> mnesia:write( rabbit_durable_queue, rabbit_queue_decorator:set( - Q#amqqueue{policy = match(Name, Policies)}), write) + Q#amqqueue{policy = match(Name, Policies), + operator_policy = match(Name, OpPolicies)}), + write) end) || Q = #amqqueue{name = Name} <- Qs], ok. @@ -133,17 +198,23 @@ invalid_file() -> %%---------------------------------------------------------------------------- +parse_set_op(VHost, Name, Pattern, Definition, Priority, ApplyTo) -> + parse_set(<<"operator_policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo). + parse_set(VHost, Name, Pattern, Definition, Priority, ApplyTo) -> + parse_set(<<"policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo). + +parse_set(Type, VHost, Name, Pattern, Definition, Priority, ApplyTo) -> try list_to_integer(Priority) of - Num -> parse_set0(VHost, Name, Pattern, Definition, Num, ApplyTo) + Num -> parse_set0(Type, VHost, Name, Pattern, Definition, Num, ApplyTo) catch error:badarg -> {error, "~p priority must be a number", [Priority]} end. -parse_set0(VHost, Name, Pattern, Defn, Priority, ApplyTo) -> +parse_set0(Type, VHost, Name, Pattern, Defn, Priority, ApplyTo) -> case rabbit_misc:json_decode(Defn) of {ok, JSON} -> - set0(VHost, Name, + set0(Type, VHost, Name, [{<<"pattern">>, list_to_binary(Pattern)}, {<<"definition">>, rabbit_misc:json_to_term(JSON)}, {<<"priority">>, Priority}, @@ -152,7 +223,13 @@ parse_set0(VHost, Name, Pattern, Defn, Priority, ApplyTo) -> {error_string, "JSON decoding error"} end. +set_op(VHost, Name, Pattern, Definition, Priority, ApplyTo) -> + set(<<"operator_policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo). + set(VHost, Name, Pattern, Definition, Priority, ApplyTo) -> + set(<<"policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo). + +set(Type, VHost, Name, Pattern, Definition, Priority, ApplyTo) -> PolicyProps = [{<<"pattern">>, Pattern}, {<<"definition">>, Definition}, {<<"priority">>, case Priority of @@ -163,20 +240,47 @@ set(VHost, Name, Pattern, Definition, Priority, ApplyTo) -> undefined -> <<"all">>; _ -> ApplyTo end}], - set0(VHost, Name, PolicyProps). + set0(Type, VHost, Name, PolicyProps). -set0(VHost, Name, Term) -> - rabbit_runtime_parameters:set_any(VHost, <<"policy">>, Name, Term, none). +set0(Type, VHost, Name, Term) -> + rabbit_runtime_parameters:set_any(VHost, Type, Name, Term, none). + +delete_op(VHost, Name) -> + rabbit_runtime_parameters:clear_any(VHost, <<"operator_policy">>, Name). delete(VHost, Name) -> rabbit_runtime_parameters:clear_any(VHost, <<"policy">>, Name). +lookup_op(VHost, Name) -> + case rabbit_runtime_parameters:lookup(VHost, <<"operator_policy">>, Name) of + not_found -> not_found; + P -> p(P, fun ident/1) + end. + lookup(VHost, Name) -> case rabbit_runtime_parameters:lookup(VHost, <<"policy">>, Name) of not_found -> not_found; P -> p(P, fun ident/1) end. +list_op() -> + list_op('_'). + +list_op(VHost) -> + list0_op(VHost, fun ident/1). + +list_formatted_op(VHost) -> + order_policies(list0_op(VHost, fun format/1)). + +list_formatted_op(VHost, Ref, AggregatorPid) -> + rabbit_control_misc:emitting_map(AggregatorPid, Ref, + fun(P) -> P end, list_formatted_op(VHost)). + +list0_op(VHost, DefnFun) -> + [p(P, DefnFun) + || P <- rabbit_runtime_parameters:list(VHost, <<"operator_policy">>)]. + + list() -> list('_'). @@ -194,8 +298,7 @@ list0(VHost, DefnFun) -> [p(P, DefnFun) || P <- rabbit_runtime_parameters:list(VHost, <<"policy">>)]. order_policies(PropList) -> - lists:sort(fun (A, B) -> pget(priority, A) < pget(priority, B) end, - PropList). + lists:sort(fun (A, B) -> not sort_pred(A, B) end, PropList). p(Parameter, DefnFun) -> Value = pget(value, Parameter), @@ -218,14 +321,23 @@ info_keys() -> [vhost, name, 'apply-to', pattern, definition, priority]. validate(_VHost, <<"policy">>, Name, Term, _User) -> rabbit_parameter_validation:proplist( - Name, policy_validation(), Term). + Name, policy_validation(), Term); +validate(_VHost, <<"operator_policy">>, Name, Term, _User) -> + rabbit_parameter_validation:proplist( + Name, operator_policy_validation(), Term). notify(VHost, <<"policy">>, Name, Term) -> rabbit_event:notify(policy_set, [{name, Name}, {vhost, VHost} | Term]), + update_policies(VHost); +notify(VHost, <<"operator_policy">>, Name, Term) -> + rabbit_event:notify(policy_set, [{name, Name}, {vhost, VHost} | Term]), update_policies(VHost). notify_clear(VHost, <<"policy">>, Name) -> rabbit_event:notify(policy_cleared, [{name, Name}, {vhost, VHost}]), + update_policies(VHost); +notify_clear(VHost, <<"operator_policy">>, Name) -> + rabbit_event:notify(operator_policy_cleared, [{name, Name}, {vhost, VHost}]), update_policies(VHost). %%---------------------------------------------------------------------------- @@ -239,50 +351,64 @@ update_policies(VHost) -> Tabs = [rabbit_queue, rabbit_durable_queue, rabbit_exchange, rabbit_durable_exchange], {Xs, Qs} = rabbit_misc:execute_mnesia_transaction( - fun() -> - [mnesia:lock({table, T}, write) || T <- Tabs], %% [1] - case catch list(VHost) of - {'EXIT', {throw, {error, {no_such_vhost, _}}}} -> - {[], []}; %% [2] - {'EXIT', Exit} -> - exit(Exit); - Policies -> - {[update_exchange(X, Policies) || - X <- rabbit_exchange:list(VHost)], - [update_queue(Q, Policies) || - Q <- rabbit_amqqueue:list(VHost)]} - end - end), + fun() -> + [mnesia:lock({table, T}, write) || T <- Tabs], %% [1] + case catch {list(VHost), list_op(VHost)} of + {'EXIT', {throw, {error, {no_such_vhost, _}}}} -> + {[], []}; %% [2] + {'EXIT', Exit} -> + exit(Exit); + {Policies, OpPolicies} -> + {[update_exchange(X, Policies, OpPolicies) || + X <- rabbit_exchange:list(VHost)], + [update_queue(Q, Policies, OpPolicies) || + Q <- rabbit_amqqueue:list(VHost)]} + end + end), [catch notify(X) || X <- Xs], [catch notify(Q) || Q <- Qs], ok. -update_exchange(X = #exchange{name = XName, policy = OldPolicy}, Policies) -> - case match(XName, Policies) of - OldPolicy -> no_change; - NewPolicy -> case rabbit_exchange:update( - XName, fun (X0) -> - rabbit_exchange_decorator:set( - X0 #exchange{policy = NewPolicy}) - end) of - #exchange{} = X1 -> {X, X1}; - not_found -> {X, X } - end +update_exchange(X = #exchange{name = XName, + policy = OldPolicy, + operator_policy = OldOpPolicy}, + Policies, OpPolicies) -> + case {match(XName, Policies), match(XName, OpPolicies)} of + {OldPolicy, OldOpPolicy} -> no_change; + {NewPolicy, NewOpPolicy} -> + NewExchange = rabbit_exchange:update( + XName, + fun(X0) -> + rabbit_exchange_decorator:set( + X0 #exchange{policy = NewPolicy, + operator_policy = NewOpPolicy}) + end), + case NewExchange of + #exchange{} = X1 -> {X, X1}; + not_found -> {X, X } + end end. -update_queue(Q = #amqqueue{name = QName, policy = OldPolicy}, Policies) -> - case match(QName, Policies) of - OldPolicy -> no_change; - NewPolicy -> case rabbit_amqqueue:update( - QName, fun(Q1) -> - rabbit_queue_decorator:set( - Q1#amqqueue{policy = NewPolicy, - policy_version = - Q1#amqqueue.policy_version + 1 }) - end) of - #amqqueue{} = Q1 -> {Q, Q1}; - not_found -> {Q, Q } - end +update_queue(Q = #amqqueue{name = QName, + policy = OldPolicy, + operator_policy = OldOpPolicy}, + Policies, OpPolicies) -> + case {match(QName, Policies), match(QName, OpPolicies)} of + {OldPolicy, OldOpPolicy} -> no_change; + {NewPolicy, NewOpPolicy} -> + NewQueue = rabbit_amqqueue:update( + QName, + fun(Q1) -> + rabbit_queue_decorator:set( + Q1#amqqueue{policy = NewPolicy, + operator_policy = NewOpPolicy, + policy_version = + Q1#amqqueue.policy_version + 1 }) + end), + case NewQueue of + #amqqueue{} = Q1 -> {Q, Q1}; + not_found -> {Q, Q } + end end. notify(no_change)-> @@ -293,11 +419,14 @@ notify({Q1 = #amqqueue{}, Q2 = #amqqueue{}}) -> rabbit_amqqueue:policy_changed(Q1, Q2). match(Name, Policies) -> - case lists:sort(fun sort_pred/2, [P || P <- Policies, matches(Name, P)]) of - [] -> undefined; - [Policy | _Rest] -> Policy + case match_all(Name, Policies) of + [] -> undefined; + [Policy | _] -> Policy end. +match_all(Name, Policies) -> + lists:sort(fun sort_pred/2, [P || P <- Policies, matches(Name, P)]). + matches(#resource{name = Name, kind = Kind, virtual_host = VHost}, Policy) -> matches_type(Kind, pget('apply-to', Policy)) andalso match =:= re:run(Name, pget(pattern, Policy), [{capture, none}]) andalso @@ -313,17 +442,29 @@ sort_pred(A, B) -> pget(priority, A) >= pget(priority, B). %%---------------------------------------------------------------------------- +operator_policy_validation() -> + [{<<"priority">>, fun rabbit_parameter_validation:number/2, mandatory}, + {<<"pattern">>, fun rabbit_parameter_validation:regex/2, mandatory}, + {<<"apply-to">>, fun apply_to_validation/2, optional}, + {<<"definition">>, fun validation_op/2, mandatory}]. + policy_validation() -> [{<<"priority">>, fun rabbit_parameter_validation:number/2, mandatory}, {<<"pattern">>, fun rabbit_parameter_validation:regex/2, mandatory}, {<<"apply-to">>, fun apply_to_validation/2, optional}, {<<"definition">>, fun validation/2, mandatory}]. -validation(_Name, []) -> +validation_op(Name, Terms) -> + validation(Name, Terms, operator_policy_validator). + +validation(Name, Terms) -> + validation(Name, Terms, policy_validator). + +validation(_Name, [], _Validator) -> {error, "no policy provided", []}; -validation(_Name, Terms) when is_list(Terms) -> +validation(_Name, Terms, Validator) when is_list(Terms) -> {Keys, Modules} = lists:unzip( - rabbit_registry:lookup_all(policy_validator)), + rabbit_registry:lookup_all(Validator)), [] = dups(Keys), %% ASSERTION Validators = lists:zipwith(fun (M, K) -> {M, a2b(K)} end, Modules, Keys), case is_proplist(Terms) of @@ -334,7 +475,7 @@ validation(_Name, Terms) when is_list(Terms) -> end; false -> {error, "definition must be a dictionary: ~p", [Terms]} end; -validation(_Name, Term) -> +validation(_Name, Term, _Validator) -> {error, "parse error while reading policy: ~p", [Term]}. validation0(Validators, Terms) -> diff --git a/src/rabbit_policy_merge_strategy.erl b/src/rabbit_policy_merge_strategy.erl new file mode 100644 index 0000000000..55ad87ccac --- /dev/null +++ b/src/rabbit_policy_merge_strategy.erl @@ -0,0 +1,28 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_policy_merge_strategy). + +-behaviour(rabbit_registry_class). + +-export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). + +-callback merge_policy_value(binary(), Value, Value) -> + Value + when Value :: term(). + +added_to_rabbit_registry(_Type, _ModuleName) -> ok. +removed_from_rabbit_registry(_Type) -> ok. diff --git a/src/rabbit_prelaunch.erl b/src/rabbit_prelaunch.erl index 569a8d6c5a..fd17860432 100644 --- a/src/rabbit_prelaunch.erl +++ b/src/rabbit_prelaunch.erl @@ -75,25 +75,39 @@ duplicate_node_check(NodeName, NodeHost) -> end. dist_port_set_check() -> - case os:getenv("RABBITMQ_CONFIG_FILE") of - false -> + case get_config(os:getenv("RABBITMQ_CONFIG_FILE")) of + {ok, [Config]} -> + Kernel = pget(kernel, Config, []), + case {pget(inet_dist_listen_min, Kernel, none), + pget(inet_dist_listen_max, Kernel, none)} of + {none, none} -> ok; + _ -> rabbit_misc:quit(?DO_NOT_SET_DIST_PORT) + end; + {ok, _} -> ok; - File -> - case file:consult(File ++ ".config") of - {ok, [Config]} -> - Kernel = pget(kernel, Config, []), - case {pget(inet_dist_listen_min, Kernel, none), - pget(inet_dist_listen_max, Kernel, none)} of - {none, none} -> ok; - _ -> rabbit_misc:quit(?DO_NOT_SET_DIST_PORT) - end; - {ok, _} -> - ok; - {error, _} -> - ok + {error, _} -> + ok + end. + +get_config(File) -> + case consult_file(File) of + {ok, Contents} -> {ok, Contents}; + {error, _} -> + case rabbit_config:get_advanced_config() of + none -> {error, enoent}; + FileName -> file:consult(FileName) end end. +consult_file(false) -> {error, nofile}; +consult_file(File) -> + FileName = case filename:extension(File) of + "" -> File ++ ".config"; + ".config" -> File; + _ -> "" + end, + file:consult(FileName). + dist_port_range_check() -> case os:getenv("RABBITMQ_DIST_PORT") of false -> ok; diff --git a/src/rabbit_queue_consumers.erl b/src/rabbit_queue_consumers.erl index a8002398e7..cd58de95dd 100644 --- a/src/rabbit_queue_consumers.erl +++ b/src/rabbit_queue_consumers.erl @@ -96,7 +96,7 @@ new() -> #state{consumers = priority_queue:new(), use = {active, - time_compat:monotonic_time(micro_seconds), + erlang:monotonic_time(micro_seconds), 1.0}}. max_active_priority(#state{consumers = Consumers}) -> @@ -346,9 +346,9 @@ drain_mode(true) -> drain; drain_mode(false) -> manual. utilisation(#state{use = {active, Since, Avg}}) -> - use_avg(time_compat:monotonic_time(micro_seconds) - Since, 0, Avg); + use_avg(erlang:monotonic_time(micro_seconds) - Since, 0, Avg); utilisation(#state{use = {inactive, Since, Active, Avg}}) -> - use_avg(Active, time_compat:monotonic_time(micro_seconds) - Since, Avg). + use_avg(Active, erlang:monotonic_time(micro_seconds) - Since, Avg). %%---------------------------------------------------------------------------- @@ -455,10 +455,10 @@ update_use({inactive, _, _, _} = CUInfo, inactive) -> update_use({active, _, _} = CUInfo, active) -> CUInfo; update_use({active, Since, Avg}, inactive) -> - Now = time_compat:monotonic_time(micro_seconds), + Now = erlang:monotonic_time(micro_seconds), {inactive, Now, Now - Since, Avg}; update_use({inactive, Since, Active, Avg}, active) -> - Now = time_compat:monotonic_time(micro_seconds), + Now = erlang:monotonic_time(micro_seconds), {active, Now, use_avg(Active, Now - Since, Avg)}. use_avg(0, 0, Avg) -> diff --git a/src/rabbit_queue_index.erl b/src/rabbit_queue_index.erl index 6a14854882..8b96bbffbd 100644 --- a/src/rabbit_queue_index.erl +++ b/src/rabbit_queue_index.erl @@ -197,13 +197,13 @@ -type segment() :: ('undefined' | #segment { num :: non_neg_integer(), path :: file:filename(), - journal_entries :: ?ARRAY_TYPE(), - entries_to_segment :: ?ARRAY_TYPE(), + journal_entries :: array:array(), + entries_to_segment :: array:array(), unacked :: non_neg_integer() }). -type seq_id() :: integer(). --type seg_dict() :: {?DICT_TYPE(), [segment()]}. --type on_sync_fun() :: fun ((?GB_SET_TYPE()) -> ok). +-type seg_dict() :: {dict:dict(), [segment()]}. +-type on_sync_fun() :: fun ((gb_sets:set()) -> ok). -type qistate() :: #qistate { dir :: file:filename(), segments :: 'undefined' | seg_dict(), journal_handle :: hdl(), @@ -211,8 +211,8 @@ max_journal_entries :: non_neg_integer(), on_sync :: on_sync_fun(), on_sync_msg :: on_sync_fun(), - unconfirmed :: ?GB_SET_TYPE(), - unconfirmed_msg :: ?GB_SET_TYPE(), + unconfirmed :: gb_sets:set(), + unconfirmed_msg :: gb_sets:set(), pre_publish_cache :: list(), delivered_cache :: list() }. diff --git a/src/rabbit_queue_location_random.erl b/src/rabbit_queue_location_random.erl index 2579cbb2b1..73d509bf33 100644 --- a/src/rabbit_queue_location_random.erl +++ b/src/rabbit_queue_location_random.erl @@ -39,6 +39,6 @@ description() -> queue_master_location(#amqqueue{}) -> Cluster = rabbit_queue_master_location_misc:all_nodes(), - RandomPos = erlang:phash2(time_compat:monotonic_time(), length(Cluster)), + RandomPos = erlang:phash2(erlang:monotonic_time(), length(Cluster)), MasterNode = lists:nth(RandomPos + 1, Cluster), {ok, MasterNode}. diff --git a/src/rabbit_registry.erl b/src/rabbit_registry.erl deleted file mode 100644 index 0428c3533f..0000000000 --- a/src/rabbit_registry.erl +++ /dev/null @@ -1,162 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. -%% - --module(rabbit_registry). - --behaviour(gen_server). - --export([start_link/0]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --export([register/3, unregister/2, - binary_to_type/1, lookup_module/2, lookup_all/1]). - --define(SERVER, ?MODULE). --define(ETS_NAME, ?MODULE). - --spec start_link() -> rabbit_types:ok_pid_or_error(). --spec register(atom(), binary(), atom()) -> 'ok'. --spec unregister(atom(), binary()) -> 'ok'. --spec binary_to_type - (binary()) -> atom() | rabbit_types:error('not_found'). --spec lookup_module - (atom(), atom()) -> rabbit_types:ok_or_error2(atom(), 'not_found'). --spec lookup_all(atom()) -> [{atom(), atom()}]. - -%%--------------------------------------------------------------------------- - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%%--------------------------------------------------------------------------- - -register(Class, TypeName, ModuleName) -> - gen_server:call(?SERVER, {register, Class, TypeName, ModuleName}, infinity). - -unregister(Class, TypeName) -> - gen_server:call(?SERVER, {unregister, Class, TypeName}, infinity). - -%% This is used with user-supplied arguments (e.g., on exchange -%% declare), so we restrict it to existing atoms only. This means it -%% can throw a badarg, indicating that the type cannot have been -%% registered. -binary_to_type(TypeBin) when is_binary(TypeBin) -> - case catch list_to_existing_atom(binary_to_list(TypeBin)) of - {'EXIT', {badarg, _}} -> {error, not_found}; - TypeAtom -> TypeAtom - end. - -lookup_module(Class, T) when is_atom(T) -> - case ets:lookup(?ETS_NAME, {Class, T}) of - [{_, Module}] -> - {ok, Module}; - [] -> - {error, not_found} - end. - -lookup_all(Class) -> - [{K, V} || [K, V] <- ets:match(?ETS_NAME, {{Class, '$1'}, '$2'})]. - -%%--------------------------------------------------------------------------- - -internal_binary_to_type(TypeBin) when is_binary(TypeBin) -> - list_to_atom(binary_to_list(TypeBin)). - -internal_register(Class, TypeName, ModuleName) - when is_atom(Class), is_binary(TypeName), is_atom(ModuleName) -> - ok = sanity_check_module(class_module(Class), ModuleName), - RegArg = {{Class, internal_binary_to_type(TypeName)}, ModuleName}, - true = ets:insert(?ETS_NAME, RegArg), - conditional_register(RegArg), - ok. - -internal_unregister(Class, TypeName) -> - UnregArg = {Class, internal_binary_to_type(TypeName)}, - conditional_unregister(UnregArg), - true = ets:delete(?ETS_NAME, UnregArg), - ok. - -%% register exchange decorator route callback only when implemented, -%% in order to avoid unnecessary decorator calls on the fast -%% publishing path -conditional_register({{exchange_decorator, Type}, ModuleName}) -> - case erlang:function_exported(ModuleName, route, 2) of - true -> true = ets:insert(?ETS_NAME, - {{exchange_decorator_route, Type}, - ModuleName}); - false -> ok - end; -conditional_register(_) -> - ok. - -conditional_unregister({exchange_decorator, Type}) -> - true = ets:delete(?ETS_NAME, {exchange_decorator_route, Type}), - ok; -conditional_unregister(_) -> - ok. - -sanity_check_module(ClassModule, Module) -> - case catch lists:member(ClassModule, - lists:flatten( - [Bs || {Attr, Bs} <- - Module:module_info(attributes), - Attr =:= behavior orelse - Attr =:= behaviour])) of - {'EXIT', {undef, _}} -> {error, not_module}; - false -> {error, {not_type, ClassModule}}; - true -> ok - end. - -class_module(exchange) -> rabbit_exchange_type; -class_module(auth_mechanism) -> rabbit_auth_mechanism; -class_module(runtime_parameter) -> rabbit_runtime_parameter; -class_module(exchange_decorator) -> rabbit_exchange_decorator; -class_module(queue_decorator) -> rabbit_queue_decorator; -class_module(policy_validator) -> rabbit_policy_validator; -class_module(ha_mode) -> rabbit_mirror_queue_mode; -class_module(channel_interceptor) -> rabbit_channel_interceptor; -class_module(queue_master_locator)-> rabbit_queue_master_locator. - -%%--------------------------------------------------------------------------- - -init([]) -> - ?ETS_NAME = ets:new(?ETS_NAME, [protected, set, named_table]), - {ok, none}. - -handle_call({register, Class, TypeName, ModuleName}, _From, State) -> - ok = internal_register(Class, TypeName, ModuleName), - {reply, ok, State}; - -handle_call({unregister, Class, TypeName}, _From, State) -> - ok = internal_unregister(Class, TypeName), - {reply, ok, State}; - -handle_call(Request, _From, State) -> - {stop, {unhandled_call, Request}, State}. - -handle_cast(Request, State) -> - {stop, {unhandled_cast, Request}, State}. - -handle_info(Message, State) -> - {stop, {unhandled_info, Message}, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/rabbit_sasl_report_file_h.erl b/src/rabbit_sasl_report_file_h.erl deleted file mode 100644 index 9c6d7657f2..0000000000 --- a/src/rabbit_sasl_report_file_h.erl +++ /dev/null @@ -1,102 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. -%% - --module(rabbit_sasl_report_file_h). --include("rabbit.hrl"). - --behaviour(gen_event). - --export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2, - code_change/3]). - --import(rabbit_error_logger_file_h, [safe_handle_event/3]). - -%% rabbit_sasl_report_file_h is a wrapper around the sasl_report_file_h -%% module because the original's init/1 does not match properly -%% with the result of closing the old handler when swapping handlers. -%% The first init/1 additionally allows for simple log rotation -%% when the suffix is not the empty string. -%% The original init/1 also opened the file in 'write' mode, thus -%% overwriting old logs. To remedy this, init/1 from -%% lib/sasl/src/sasl_report_file_h.erl from R14B3 was copied as -%% init_file/1 and changed so that it opens the file in 'append' mode. - -%% Used only when swapping handlers and performing -%% log rotation -init({{File, ""}, _}) -> - init(File); -init({{File, Suffix}, []}) -> - case rabbit_file:append_file(File, Suffix) of - ok -> file:delete(File), - ok; - {error, Error} -> - rabbit_log:error("Failed to append contents of " - "sasl log file '~s' to '~s':~n~p~n", - [File, [File, Suffix], Error]) - end, - init(File); -%% Used only when swapping handlers and the original handler -%% failed to terminate or was never installed -init({{File, _}, error}) -> - init(File); -%% Used only when swapping handlers without -%% doing any log rotation -init({File, []}) -> - init(File); -init({File, _Type} = FileInfo) -> - rabbit_file:ensure_parent_dirs_exist(File), - init_file(FileInfo); -init(File) -> - rabbit_file:ensure_parent_dirs_exist(File), - init_file({File, sasl_error_logger_type()}). - -init_file({File, Type}) -> - process_flag(trap_exit, true), - case file:open(File, [append]) of - {ok,Fd} -> {ok, {Fd, File, Type}}; - Error -> Error - end. - -handle_event(Event, State) -> - safe_handle_event(fun handle_event0/2, Event, State). - -handle_event0(Event, State) -> - sasl_report_file_h:handle_event( - truncate:log_event(Event, ?LOG_TRUNC), State). - -handle_info(Info, State) -> - sasl_report_file_h:handle_info(Info, State). - -handle_call(Call, State) -> - sasl_report_file_h:handle_call(Call, State). - -terminate(Reason, State) -> - sasl_report_file_h:terminate(Reason, State). - -code_change(_OldVsn, State, _Extra) -> - %% There is no sasl_report_file_h:code_change/3 - {ok, State}. - -%%---------------------------------------------------------------------- - -sasl_error_logger_type() -> - case application:get_env(sasl, errlog_type) of - {ok, error} -> error; - {ok, progress} -> progress; - {ok, all} -> all; - {ok, Bad} -> throw({error, {wrong_errlog_type, Bad}}); - _ -> all - end. diff --git a/src/rabbit_table.erl b/src/rabbit_table.erl index 3909096964..1bb19b23da 100644 --- a/src/rabbit_table.erl +++ b/src/rabbit_table.erl @@ -50,8 +50,20 @@ create() -> Tab, TabDef1, Reason}}) end end, definitions()), + ensure_secondary_indexes(), ok. +%% Sets up secondary indexes in a blank node database. +ensure_secondary_indexes() -> + ensure_secondary_index(rabbit_queue, vhost), + ok. + +ensure_secondary_index(Table, Field) -> + case mnesia:add_table_index(Table, Field) of + {atomic, ok} -> ok; + {aborted, {already_exists, Table, _}} -> ok + end. + %% The sequence in which we delete the schema and then the other %% tables is important: if we delete the schema first when moving to %% RAM mnesia will loudly complain since it doesn't make much sense to diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index 8609a0e424..a53ad0c8f9 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -24,6 +24,7 @@ -rabbit_upgrade({remove_user_scope, mnesia, []}). -rabbit_upgrade({hash_passwords, mnesia, []}). -rabbit_upgrade({add_ip_to_listener, mnesia, []}). +-rabbit_upgrade({add_opts_to_listener, mnesia, [add_ip_to_listener]}). -rabbit_upgrade({internal_exchanges, mnesia, []}). -rabbit_upgrade({user_to_internal_user, mnesia, [hash_passwords]}). -rabbit_upgrade({topic_trie, mnesia, []}). @@ -52,13 +53,18 @@ -rabbit_upgrade({queue_state, mnesia, [down_slave_nodes]}). -rabbit_upgrade({recoverable_slaves, mnesia, [queue_state]}). -rabbit_upgrade({policy_version, mnesia, [recoverable_slaves]}). +-rabbit_upgrade({slave_pids_pending_shutdown, mnesia, [policy_version]}). -rabbit_upgrade({user_password_hashing, mnesia, [hash_passwords]}). +-rabbit_upgrade({operator_policies, mnesia, [slave_pids_pending_shutdown, internal_system_x]}). +-rabbit_upgrade({vhost_limits, mnesia, []}). +-rabbit_upgrade({queue_vhost_field, mnesia, [operator_policies]}). %% ------------------------------------------------------------------- -spec remove_user_scope() -> 'ok'. -spec hash_passwords() -> 'ok'. -spec add_ip_to_listener() -> 'ok'. +-spec add_opts_to_listener() -> 'ok'. -spec internal_exchanges() -> 'ok'. -spec user_to_internal_user() -> 'ok'. -spec topic_trie() -> 'ok'. @@ -85,9 +91,24 @@ -spec queue_state() -> 'ok'. -spec recoverable_slaves() -> 'ok'. -spec user_password_hashing() -> 'ok'. +-spec vhost_limits() -> 'ok'. +-spec operator_policies() -> 'ok'. +-spec queue_vhost_field() -> 'ok'. + %%-------------------------------------------------------------------- +%% replaces vhost.dummy (used to avoid having a single-field record +%% which Mnesia doesn't like) with vhost.limits (which is actually +%% used) +vhost_limits() -> + transform( + rabbit_vhost, + fun ({vhost, VHost, _Dummy}) -> + {vhost, VHost, undefined} + end, + [virtual_host, limits]). + %% It's a bad idea to use records or record_info here, even for the %% destination form. Because in the future, the destination form of %% your current transform may not match the record any more, and it @@ -123,6 +144,14 @@ add_ip_to_listener() -> end, [node, protocol, host, ip_address, port]). +add_opts_to_listener() -> + transform( + rabbit_listener, + fun ({listener, Node, Protocol, Host, IP, Port}) -> + {listener, Node, Protocol, Host, IP, Port, []} + end, + [node, protocol, host, ip_address, port, opts]). + internal_exchanges() -> Tables = [rabbit_exchange, rabbit_durable_exchange], AddInternalFun = @@ -452,6 +481,77 @@ policy_version(Table) -> sync_slave_pids, recoverable_slaves, policy, gm_pids, decorators, state, policy_version]). +slave_pids_pending_shutdown() -> + ok = slave_pids_pending_shutdown(rabbit_queue), + ok = slave_pids_pending_shutdown(rabbit_durable_queue). + +slave_pids_pending_shutdown(Table) -> + transform( + Table, + fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, + Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, + State, PolicyVersion}) -> + {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, + Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, + State, PolicyVersion, []} + end, + [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, + sync_slave_pids, recoverable_slaves, policy, gm_pids, decorators, state, + policy_version, slave_pids_pending_shutdown]). + +operator_policies() -> + ok = exchange_operator_policies(rabbit_exchange), + ok = exchange_operator_policies(rabbit_durable_exchange), + ok = queue_operator_policies(rabbit_queue), + ok = queue_operator_policies(rabbit_durable_queue). + +exchange_operator_policies(Table) -> + transform( + Table, + fun ({exchange, Name, Type, Dur, AutoDel, Internal, + Args, Scratches, Policy, Decorators}) -> + {exchange, Name, Type, Dur, AutoDel, Internal, + Args, Scratches, Policy, undefined, Decorators} + end, + [name, type, durable, auto_delete, internal, arguments, scratches, policy, + operator_policy, decorators]). + +queue_operator_policies(Table) -> + transform( + Table, + fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, + Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, + State, PolicyVersion, SlavePidsPendingShutdown}) -> + {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, + Pid, SlavePids, SyncSlavePids, DSN, Policy, undefined, GmPids, + Decorators, State, PolicyVersion, SlavePidsPendingShutdown} + end, + [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, + sync_slave_pids, recoverable_slaves, policy, operator_policy, + gm_pids, decorators, state, policy_version, slave_pids_pending_shutdown]). + + +queue_vhost_field() -> + ok = queue_vhost_field(rabbit_queue), + ok = queue_vhost_field(rabbit_durable_queue), + {atomic, ok} = mnesia:add_table_index(rabbit_queue, vhost), + {atomic, ok} = mnesia:add_table_index(rabbit_durable_queue, vhost), + ok. + +queue_vhost_field(Table) -> + transform( + Table, + fun ({amqqueue, Name = {resource, VHost, queue, _QName}, Durable, AutoDelete, ExclusiveOwner, Arguments, + Pid, SlavePids, SyncSlavePids, DSN, Policy, OperatorPolicy, GmPids, Decorators, + State, PolicyVersion, SlavePidsPendingShutdown}) -> + {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, + Pid, SlavePids, SyncSlavePids, DSN, Policy, OperatorPolicy, GmPids, Decorators, + State, PolicyVersion, SlavePidsPendingShutdown, VHost} + end, + [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, + sync_slave_pids, recoverable_slaves, policy, operator_policy, + gm_pids, decorators, state, policy_version, slave_pids_pending_shutdown, vhost]). + %% Prior to 3.6.0, passwords were hashed using MD5, this populates %% existing records with said default. Users created with 3.6.0+ will %% have internal_user.hashing_algorithm populated by the internal diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index dd92256146..bbec4c749d 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -396,10 +396,10 @@ out_counter :: non_neg_integer(), in_counter :: non_neg_integer(), rates :: rates(), - msgs_on_disk :: ?GB_SET_TYPE(), - msg_indices_on_disk :: ?GB_SET_TYPE(), - unconfirmed :: ?GB_SET_TYPE(), - confirmed :: ?GB_SET_TYPE(), + msgs_on_disk :: gb_sets:set(), + msg_indices_on_disk :: gb_sets:set(), + unconfirmed :: gb_sets:set(), + confirmed :: gb_sets:set(), ack_out_counter :: non_neg_integer(), ack_in_counter :: non_neg_integer(), disk_read_count :: non_neg_integer(), @@ -797,7 +797,7 @@ update_rates(State = #vqstate{ in_counter = InCount, ack_in = AckInRate, ack_out = AckOutRate, timestamp = TS }}) -> - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), Rates = #rates { in = update_rate(Now, TS, InCount, InRate), out = update_rate(Now, TS, OutCount, OutRate), @@ -812,7 +812,7 @@ update_rates(State = #vqstate{ in_counter = InCount, rates = Rates }. update_rate(Now, TS, Count, Rate) -> - Time = time_compat:convert_time_unit(Now - TS, native, micro_seconds) / + Time = erlang:convert_time_unit(Now - TS, native, micro_seconds) / ?MICROS_PER_SECOND, if Time == 0 -> Rate; @@ -1310,7 +1310,7 @@ init(IsDurable, IndexState, DeltaCount, DeltaBytes, Terms, count = DeltaCount1, end_seq_id = NextSeqId }) end, - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), IoBatchSize = rabbit_misc:get_env(rabbit, msg_store_io_batch_size, ?IO_BATCH_SIZE), diff --git a/src/rabbit_vhost.erl b/src/rabbit_vhost.erl index df2f8423b4..01f1046fb8 100644 --- a/src/rabbit_vhost.erl +++ b/src/rabbit_vhost.erl @@ -20,11 +20,14 @@ %%---------------------------------------------------------------------------- --export([add/1, delete/1, exists/1, list/0, with/2, assert/1]). +-export([add/1, delete/1, exists/1, list/0, with/2, assert/1, update/2, + set_limits/2, limits_of/1]). -export([info/1, info/2, info_all/0, info_all/1, info_all/2, info_all/3]). + -spec add(rabbit_types:vhost()) -> 'ok'. -spec delete(rabbit_types:vhost()) -> 'ok'. +-spec update(rabbit_types:vhost(), rabbit_misc:thunk(A)) -> A. -spec exists(rabbit_types:vhost()) -> boolean(). -spec list() -> [rabbit_types:vhost()]. -spec with(rabbit_types:vhost(), rabbit_misc:thunk(A)) -> A. @@ -138,6 +141,32 @@ assert(VHostPath) -> case exists(VHostPath) of false -> throw({error, {no_such_vhost, VHostPath}}) end. +update(VHostPath, Fun) -> + case mnesia:read({rabbit_vhost, VHostPath}) of + [] -> + mnesia:abort({no_such_vhost, VHostPath}); + [V] -> + V1 = Fun(V), + ok = mnesia:write(rabbit_vhost, V1, write), + V1 + end. + +limits_of(VHostPath) when is_binary(VHostPath) -> + assert(VHostPath), + case mnesia:dirty_read({rabbit_vhost, VHostPath}) of + [] -> + mnesia:abort({no_such_vhost, VHostPath}); + [#vhost{limits = Limits}] -> + Limits + end; +limits_of(#vhost{virtual_host = Name}) -> + limits_of(Name). + +set_limits(VHost = #vhost{}, undefined) -> + VHost#vhost{limits = undefined}; +set_limits(VHost = #vhost{}, Limits) -> + VHost#vhost{limits = Limits}. + %%---------------------------------------------------------------------------- infos(Items, X) -> [{Item, i(Item, X)} || Item <- Items]. diff --git a/src/rabbit_vhost_limit.erl b/src/rabbit_vhost_limit.erl new file mode 100644 index 0000000000..b933c31402 --- /dev/null +++ b/src/rabbit_vhost_limit.erl @@ -0,0 +1,184 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_vhost_limit). + +-behaviour(rabbit_runtime_parameter). + +-include("rabbit.hrl"). + +-export([register/0]). +-export([parse_set/2, set/2, clear/1]). +-export([list/0, list/1]). +-export([update_limit/3, clear_limit/2, get_limit/2]). +-export([validate/5, notify/4, notify_clear/3]). +-export([connection_limit/1, queue_limit/1, + is_over_queue_limit/1, is_over_connection_limit/1]). + +-import(rabbit_misc, [pget/2, pget/3]). + +-rabbit_boot_step({?MODULE, + [{description, "vhost limit parameters"}, + {mfa, {rabbit_vhost_limit, register, []}}, + {requires, rabbit_registry}, + {enables, recovery}]}). + +%%---------------------------------------------------------------------------- + +register() -> + rabbit_registry:register(runtime_parameter, <<"vhost-limits">>, ?MODULE). + +validate(_VHost, <<"vhost-limits">>, Name, Term, _User) -> + rabbit_parameter_validation:proplist( + Name, vhost_limit_validation(), Term). + +notify(VHost, <<"vhost-limits">>, <<"limits">>, Limits) -> + rabbit_event:notify(vhost_limits_set, [{name, <<"limits">>} | Limits]), + update_vhost(VHost, Limits). + +notify_clear(VHost, <<"vhost-limits">>, <<"limits">>) -> + rabbit_event:notify(vhost_limits_cleared, [{name, <<"limits">>}]), + update_vhost(VHost, undefined). + +connection_limit(VirtualHost) -> + get_limit(VirtualHost, <<"max-connections">>). + +queue_limit(VirtualHost) -> + get_limit(VirtualHost, <<"max-queues">>). + +-spec list() -> [{rabbit_types:vhost(), rabbit_types:infos()}]. + +list() -> + case rabbit_runtime_parameters:list_component(<<"vhost-limits">>) of + [] -> []; + Params -> [ {pget(vhost, Param), pget(value, Param)} + || Param <- Params, + pget(value, Param) =/= undefined, + pget(name, Param) == <<"limits">> ] + end. + +-spec list(rabbit_types:vhost()) -> rabbit_types:infos(). + +list(VHost) -> + rabbit_runtime_parameters:value(VHost, <<"vhost-limits">>, <<"limits">>, []). + +-spec is_over_connection_limit(rabbit_types:vhost()) -> {true, non_neg_integer()} | false. + +is_over_connection_limit(VirtualHost) -> + case rabbit_vhost_limit:connection_limit(VirtualHost) of + %% no limit configured + undefined -> false; + %% with limit = 0, no connections are allowed + {ok, 0} -> {true, 0}; + {ok, Limit} when is_integer(Limit) andalso Limit > 0 -> + ConnectionCount = rabbit_connection_tracking:count_connections_in(VirtualHost), + case ConnectionCount >= Limit of + false -> false; + true -> {true, Limit} + end; + %% any negative value means "no limit". Note that parameter validation + %% will replace negative integers with 'undefined', so this is to be + %% explicit and extra defensive + {ok, Limit} when is_integer(Limit) andalso Limit < 0 -> false; + %% ignore non-integer limits + {ok, _Limit} -> false + end. + + +-spec is_over_queue_limit(rabbit_types:vhost()) -> {true, non_neg_integer()} | false. + +is_over_queue_limit(VirtualHost) -> + case queue_limit(VirtualHost) of + %% no limit configured + undefined -> false; + %% with limit = 0, no queues can be declared (perhaps not very + %% useful but consistent with the connection limit) + {ok, 0} -> {true, 0}; + {ok, Limit} when is_integer(Limit) andalso Limit > 0 -> + QueueCount = rabbit_amqqueue:count(VirtualHost), + case QueueCount >= Limit of + false -> false; + true -> {true, Limit} + end; + %% any negative value means "no limit". Note that parameter validation + %% will replace negative integers with 'undefined', so this is to be + %% explicit and extra defensive + {ok, Limit} when is_integer(Limit) andalso Limit < 0 -> false; + %% ignore non-integer limits + {ok, _Limit} -> false + end. + +%%---------------------------------------------------------------------------- + +parse_set(VHost, Defn) -> + case rabbit_misc:json_decode(Defn) of + {ok, JSON} -> + set(VHost, rabbit_misc:json_to_term(JSON)); + error -> + {error_string, "JSON decoding error"} + end. + +set(VHost, Defn) -> + rabbit_runtime_parameters:set_any(VHost, <<"vhost-limits">>, + <<"limits">>, Defn, none). + +clear(VHost) -> + rabbit_runtime_parameters:clear_any(VHost, <<"vhost-limits">>, + <<"limits">>). + +update_limit(VHost, Name, Value) -> + OldDef = case rabbit_runtime_parameters:list(VHost, <<"vhost-limits">>) of + [] -> []; + [Param] -> pget(value, Param, []) + end, + NewDef = [{Name, Value} | lists:keydelete(Name, 1, OldDef)], + set(VHost, NewDef). + +clear_limit(VHost, Name) -> + OldDef = case rabbit_runtime_parameters:list(VHost, <<"vhost-limits">>) of + [] -> []; + [Param] -> pget(value, Param, []) + end, + NewDef = lists:keydelete(Name, 1, OldDef), + set(VHost, NewDef). + +vhost_limit_validation() -> + [{<<"max-connections">>, fun rabbit_parameter_validation:integer/2, optional}, + {<<"max-queues">>, fun rabbit_parameter_validation:integer/2, optional}]. + +update_vhost(VHostName, Limits) -> + rabbit_misc:execute_mnesia_transaction( + fun() -> + rabbit_vhost:update(VHostName, + fun(VHost) -> + rabbit_vhost:set_limits(VHost, Limits) + end) + end), + ok. + +get_limit(VirtualHost, Limit) -> + case rabbit_runtime_parameters:list(VirtualHost, <<"vhost-limits">>) of + [] -> undefined; + [Param] -> case pget(value, Param) of + undefined -> undefined; + Val -> case pget(Limit, Val) of + undefined -> undefined; + %% no limit + N when N < 0 -> undefined; + N when N >= 0 -> {ok, N} + end + end + end. diff --git a/test/channel_interceptor_SUITE.erl b/test/channel_interceptor_SUITE.erl new file mode 100644 index 0000000000..0e4948ea3c --- /dev/null +++ b/test/channel_interceptor_SUITE.erl @@ -0,0 +1,113 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(channel_interceptor_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + register_interceptor + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +register_interceptor(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, register_interceptor1, [Config]). + +register_interceptor1(Config) -> + PredefinedChannels = rabbit_channel:list(), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, 0), + + QName = <<"register_interceptor-q">>, + amqp_channel:call(Ch1, #'queue.declare'{queue = QName}), + + [ChannelProc] = rabbit_channel:list() -- PredefinedChannels, + + [{interceptors, []}] = rabbit_channel:info(ChannelProc, [interceptors]), + + check_send_receive(Ch1, QName, <<"bar">>, <<"bar">>), + + ok = rabbit_registry:register(channel_interceptor, + <<"dummy interceptor">>, + dummy_interceptor), + [{interceptors, [{dummy_interceptor, undefined}]}] = + rabbit_channel:info(ChannelProc, [interceptors]), + + check_send_receive(Ch1, QName, <<"bar">>, <<"">>), + + ok = rabbit_registry:unregister(channel_interceptor, + <<"dummy interceptor">>), + [{interceptors, []}] = rabbit_channel:info(ChannelProc, [interceptors]), + + check_send_receive(Ch1, QName, <<"bar">>, <<"bar">>), + passed. + + +check_send_receive(Ch1, QName, Send, Receive) -> + amqp_channel:call(Ch1, + #'basic.publish'{routing_key = QName}, + #amqp_msg{payload = Send}), + + {#'basic.get_ok'{}, #amqp_msg{payload = Receive}} = + amqp_channel:call(Ch1, #'basic.get'{queue = QName, + no_ack = true}). diff --git a/test/channel_operation_timeout_test_queue.erl b/test/channel_operation_timeout_test_queue.erl index 0bb3f5a1c0..4407a24e7f 100644 --- a/test/channel_operation_timeout_test_queue.erl +++ b/test/channel_operation_timeout_test_queue.erl @@ -174,10 +174,10 @@ out_counter :: non_neg_integer(), in_counter :: non_neg_integer(), rates :: rates(), - msgs_on_disk :: ?GB_SET_TYPE(), - msg_indices_on_disk :: ?GB_SET_TYPE(), - unconfirmed :: ?GB_SET_TYPE(), - confirmed :: ?GB_SET_TYPE(), + msgs_on_disk :: gb_sets:set(), + msg_indices_on_disk :: gb_sets:set(), + unconfirmed :: gb_sets:set(), + confirmed :: gb_sets:set(), ack_out_counter :: non_neg_integer(), ack_in_counter :: non_neg_integer(), disk_read_count :: non_neg_integer(), @@ -559,7 +559,7 @@ update_rates(State = #vqstate{ in_counter = InCount, ack_in = AckInRate, ack_out = AckOutRate, timestamp = TS }}) -> - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), Rates = #rates { in = update_rate(Now, TS, InCount, InRate), out = update_rate(Now, TS, OutCount, OutRate), @@ -574,7 +574,7 @@ update_rates(State = #vqstate{ in_counter = InCount, rates = Rates }. update_rate(Now, TS, Count, Rate) -> - Time = time_compat:convert_time_unit(Now - TS, native, micro_seconds) / + Time = erlang:convert_time_unit(Now - TS, native, micro_seconds) / ?MICROS_PER_SECOND, if Time == 0 -> Rate; @@ -1072,7 +1072,7 @@ init(IsDurable, IndexState, DeltaCount, DeltaBytes, Terms, count = DeltaCount1, end_seq_id = NextSeqId }) end, - Now = time_compat:monotonic_time(), + Now = erlang:monotonic_time(), IoBatchSize = rabbit_misc:get_env(rabbit, msg_store_io_batch_size, ?IO_BATCH_SIZE), diff --git a/test/cluster_rename_SUITE.erl b/test/cluster_rename_SUITE.erl index 8ce29a6695..9521e04eb5 100644 --- a/test/cluster_rename_SUITE.erl +++ b/test/cluster_rename_SUITE.erl @@ -43,6 +43,12 @@ groups() -> ]} ]. +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + %% ------------------------------------------------------------------- %% Testsuite setup/teardown. %% ------------------------------------------------------------------- @@ -245,7 +251,7 @@ rename_node(Config, Nodename, Map) -> Config1. rename_node_fail(Config, Nodename, Map) -> - error = do_rename_node(Config, Nodename, Map), + {error, _, _} = do_rename_node(Config, Nodename, Map), ok. do_rename_node(Config, Nodename, Map) -> @@ -265,8 +271,8 @@ do_rename_node(Config, Nodename, Map) -> {ok, _} -> Config1 = update_config_after_rename(Config, Map1), {ok, Config1}; - {error, _, _} -> - error + {error, _, _} = Error -> + Error end. update_config_after_rename(Config, [Old, New | Rest]) -> diff --git a/test/clustering_management_SUITE.erl b/test/clustering_management_SUITE.erl index 00ddfa48a2..c40d624ddf 100644 --- a/test/clustering_management_SUITE.erl +++ b/test/clustering_management_SUITE.erl @@ -516,13 +516,13 @@ erlang_config(Config) -> assert_not_clustered(Hare), assert_not_clustered(Rabbit), - %% If we use a legacy config file, the node fails to start. + %% List of nodes [node()] is equivalent to {[node()], disk} ok = stop_app(Hare), ok = reset(Hare), ok = rpc:call(Hare, application, set_env, [rabbit, cluster_nodes, [Rabbit]]), - assert_failure(fun () -> start_app(Hare) end), - assert_not_clustered(Rabbit), + ok = start_app(Hare), + assert_clustered([Rabbit, Hare]), %% If we use an invalid node name, the node fails to start. ok = stop_app(Hare), diff --git a/test/config_schema_SUITE.erl b/test/config_schema_SUITE.erl new file mode 100644 index 0000000000..09eb4678cd --- /dev/null +++ b/test/config_schema_SUITE.erl @@ -0,0 +1,162 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(config_schema_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + run_snippets + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:run_setup_steps(Config), + DepsDir = ?config(erlang_mk_depsdir, Config1), + Schemas = filelib:wildcard(DepsDir ++ "/*/priv/schema/*.schema"), + ct:pal("Schemas ~p~n", [Schemas]), + SchemaDir = filename:join(?config(data_dir, Config1), "schema"), + file:make_dir(SchemaDir), + ct:pal("Schema DIR ~p~n", [SchemaDir]), + [ copy_to(Schema, SchemaDir) || Schema <- Schemas ], + rabbit_ct_helpers:set_config(Config1, [{schema_dir, SchemaDir}]). + +copy_to(File, Dir) -> + BaseName = filename:basename(File), + {ok, _} = file:copy(File, Dir ++ "/" ++ BaseName). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase} + ]), + Config2 = case Testcase of + run_snippets -> + ResultsDir = filename:join(?config(priv_dir, Config1), "results"), + Snippets = filename:join(?config(data_dir, Config1), + "snippets.config"), + ok = file:make_dir(ResultsDir), + rabbit_ct_helpers:set_config(Config1, [ + {results_dir, ResultsDir}, + {conf_snippets, Snippets} + ]) + end, + rabbit_ct_helpers:run_steps(Config2, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +run_snippets(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, run_snippets1, [Config]). + +run_snippets1(Config) -> + {ok, [Snippets]} = file:consult(?config(conf_snippets, Config)), + lists:map( + fun({N, S, C, P}) -> ok = test_snippet(Config, {snippet_id(N), S, []}, C, P); + ({N, S, A, C, P}) -> ok = test_snippet(Config, {snippet_id(N), S, A}, C, P) + end, + Snippets), + passed. + +snippet_id(N) when is_integer(N) -> + integer_to_list(N); +snippet_id(F) when is_float(F) -> + float_to_list(F); +snippet_id(A) when is_atom(A) -> + atom_to_list(A); +snippet_id(L) when is_list(L) -> + L. + +test_snippet(Config, Snippet, Expected, _Plugins) -> + {ConfFile, AdvancedFile} = write_snippet(Config, Snippet), + {ok, GeneratedFile} = generate_config(Config, ConfFile, AdvancedFile), + {ok, [Generated]} = file:consult(GeneratedFile), + Gen = deepsort(Generated), + Exp = deepsort(Expected), + case Exp of + Gen -> ok; + _ -> + error({config_mismatch, Snippet, Exp, Gen}) + end. + +write_snippet(Config, {Name, Conf, Advanced}) -> + ResultsDir = ?config(results_dir, Config), + file:make_dir(filename:join(ResultsDir, Name)), + ConfFile = filename:join([ResultsDir, Name, "config.conf"]), + AdvancedFile = filename:join([ResultsDir, Name, "advanced.config"]), + + file:write_file(ConfFile, Conf), + rabbit_file:write_term_file(AdvancedFile, [Advanced]), + {ConfFile, AdvancedFile}. + +generate_config(Config, ConfFile, AdvancedFile) -> + SchemaDir = ?config(schema_dir, Config), + ResultsDir = ?config(results_dir, Config), + Rabbitmqctl = ?config(rabbitmqctl_cmd, Config), + ScriptDir = filename:dirname(Rabbitmqctl), + ct:pal("ConfFile=~p ScriptDir=~p SchemaDir=~p AdvancedFile=~p", [ConfFile, ScriptDir, SchemaDir, AdvancedFile]), + rabbit_config:generate_config_file([ConfFile], ResultsDir, ScriptDir, + SchemaDir, AdvancedFile). + +deepsort(List) -> + case is_proplist(List) of + true -> + lists:keysort(1, lists:map(fun({K, V}) -> {K, deepsort(V)}; + (V) -> V end, + List)); + false -> + case is_list(List) of + true -> lists:sort(List); + false -> List + end + end. + +is_proplist([{_Key, _Val}|_] = List) -> lists:all(fun({_K, _V}) -> true; (_) -> false end, List); +is_proplist(_) -> false. diff --git a/test/config_schema_SUITE_data/certs/cacert.pem b/test/config_schema_SUITE_data/certs/cacert.pem new file mode 100644 index 0000000000..eaf6b67806 --- /dev/null +++ b/test/config_schema_SUITE_data/certs/cacert.pem @@ -0,0 +1 @@ +I'm not a certificate diff --git a/test/config_schema_SUITE_data/certs/cert.pem b/test/config_schema_SUITE_data/certs/cert.pem new file mode 100644 index 0000000000..eaf6b67806 --- /dev/null +++ b/test/config_schema_SUITE_data/certs/cert.pem @@ -0,0 +1 @@ +I'm not a certificate diff --git a/test/config_schema_SUITE_data/certs/key.pem b/test/config_schema_SUITE_data/certs/key.pem new file mode 100644 index 0000000000..eaf6b67806 --- /dev/null +++ b/test/config_schema_SUITE_data/certs/key.pem @@ -0,0 +1 @@ +I'm not a certificate diff --git a/test/config_schema_SUITE_data/rabbit-mgmt/access.log b/test/config_schema_SUITE_data/rabbit-mgmt/access.log new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/config_schema_SUITE_data/rabbit-mgmt/access.log diff --git a/test/config_schema_SUITE_data/snippets.config b/test/config_schema_SUITE_data/snippets.config new file mode 100644 index 0000000000..8c4319fa53 --- /dev/null +++ b/test/config_schema_SUITE_data/snippets.config @@ -0,0 +1,745 @@ +[ +{1, +"auth_backends.1 = internal", +[{rabbit, [{auth_backends, [rabbit_auth_backend_internal]}]}],[]} +, +{2, +"auth_backends.1 = ldap", +[{rabbit, [{auth_backends, [rabbit_auth_backend_ldap]}]}],[]} +, + +{3, +"auth_backends.1 = ldap +auth_backends.2 = internal", + +[{rabbit, [ + {auth_backends, [rabbit_auth_backend_ldap, rabbit_auth_backend_internal]} + ] + }],[]} + +, + +{4, +"auth_backends.1 = ldap +# uses module name instead of a short alias, \"http\" +auth_backends.2 = rabbit_auth_backend_http", + +[{rabbit, [{auth_backends, [rabbit_auth_backend_ldap, rabbit_auth_backend_http]}]}],[]} + +, + +{5, +"auth_backends.1.authn = internal +# uses module name because this backend is from a 3rd party +auth_backends.1.authz = rabbit_auth_backend_ip_range", +[{rabbit, [{auth_backends, [{rabbit_auth_backend_internal, rabbit_auth_backend_ip_range}]}]}],[]} +, +{6, +"auth_backends.1.authn = ldap +auth_backends.1.authz = internal", +[{rabbit, [{auth_backends, [{rabbit_auth_backend_ldap, rabbit_auth_backend_internal}]}]}],[]} +, + +{7, +"auth_backends.1.authn = ldap +auth_backends.1.authz = internal +auth_backends.2 = internal", +[{rabbit, [ + {auth_backends, [{rabbit_auth_backend_ldap, rabbit_auth_backend_internal}, + rabbit_auth_backend_internal]} + ] + }],[]} +, + + +{8, +"ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true", +[ + {rabbit, [{ssl_options, [{cacertfile, "test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}]} + ]}],[]} +, + +{9, +"listeners.tcp.default = 5673", +[{rabbit, [{tcp_listeners, [5673]}]}],[]} +, + +{10, +"listeners.ssl = none", +[{rabbit, [{ssl_listeners, []}]}],[]} +, +{11, +"num_acceptors.ssl = 1", +[{rabbit, [{num_ssl_acceptors, 1}]}],[]} +, +{12, +"default_user = guest +default_pass = guest +default_user_tags.administrator = true +default_permissions.configure = .* +default_permissions.read = .* +default_permissions.write = .*", +[{rabbit, [ +{default_user, <<"guest">>}, +{default_pass, <<"guest">>}, +{default_user_tags, [administrator]}, +{default_permissions, [<<".*">>, <<".*">>, <<".*">>]}]}],[]} +, +{13, +"autocluster.classic_config.nodes.peer1 = rabbit@hostname1 +autocluster.classic_config.nodes.peer2 = rabbit@hostname2 +autocluster.classic_config.node_type = disc", +[{rabbit, [ + {cluster_nodes, {[rabbit@hostname2,rabbit@hostname1], disc}} +]}],[]} +, +{13.1, +"autocluster.classic_config.nodes.peer1 = rabbit@hostname1 +autocluster.classic_config.nodes.peer2 = rabbit@hostname2 +autocluster.classic_config.node_type = disk", +[{rabbit, [ + {cluster_nodes, {[rabbit@hostname2,rabbit@hostname1], disc}} +]}],[]} +, +{13.2, +"autocluster.classic_config.node_type = ram", +[],[]} +, +{14, +"tcp_listen_options.backlog = 128 +tcp_listen_options.nodelay = true +tcp_listen_options.exit_on_close = false", +[{rabbit, [{tcp_listen_options, [{backlog, 128}, +{nodelay, true}, +{exit_on_close, false}]}]}],[]} +, +{15, +"auth_backends.1.authn = ldap +auth_backends.1.authz = internal +auth_backends.2 = internal", +[{rabbit,[{auth_backends, [{rabbit_auth_backend_ldap, rabbit_auth_backend_internal}, + rabbit_auth_backend_internal]}]}],[]} +, +{16, +"rabbitmq_auth_backend_ldap.servers.1 = some_server + rabbitmq_auth_backend_ldap.servers.2 = some_other_server", +[{rabbitmq_auth_backend_ldap, [{servers, ["some_server", "some_other_server"]}]}], +[rabbitmq_auth_backend_ldap]} +, +{17, +"rabbitmq_auth_backend_ldap.dn_lookup_attribute = userPrincipalName +rabbitmq_auth_backend_ldap.dn_lookup_base = DC=gopivotal,DC=com +rabbitmq_auth_backend_ldap.dn_lookup_bind = as_user", +[{rabbitmq_auth_backend_ldap, [{dn_lookup_attribute, "userPrincipalName"}, +{dn_lookup_base, "DC=gopivotal,DC=com"}, +{dn_lookup_bind, as_user}]}], +[rabbitmq_auth_backend_ldap]} +, +{18, +"rabbitmq_auth_backend_ldap.dn_lookup_bind.user_dn = username +rabbitmq_auth_backend_ldap.dn_lookup_bind.password = password", +[{rabbitmq_auth_backend_ldap, [ +{dn_lookup_bind, {"username", "password"}}]}], +[rabbitmq_auth_backend_ldap]} +, +{19, +"rabbitmq_auth_backend_ldap.other_bind = anon", +[{rabbitmq_auth_backend_ldap, [{other_bind, anon}]}], +[rabbitmq_auth_backend_ldap]} +, +{20, +"rabbitmq_auth_backend_ldap.other_bind = as_user", +[{rabbitmq_auth_backend_ldap, [{other_bind, as_user}]}], +[rabbitmq_auth_backend_ldap]} +, +{21, +"rabbitmq_auth_backend_ldap.other_bind.user_dn = username +rabbitmq_auth_backend_ldap.other_bind.password = password", +[{rabbitmq_auth_backend_ldap, [{other_bind, {"username", "password"}}]}], +[rabbitmq_auth_backend_ldap]} +, +{22, +"listeners.tcp.default = 5672 +collect_statistics_interval = 10000 +management.http_log_dir = test/config_schema_SUITE_data/rabbit-mgmt +management.rates_mode = basic", +[{rabbit, [ {tcp_listeners, [5672]}, + {collect_statistics_interval, 10000}]}, + {rabbitmq_management, [ {http_log_dir, "test/config_schema_SUITE_data/rabbit-mgmt"}, + {rates_mode, basic}]} +], +[rabbitmq_management]} +, +{23, +"management.listener.port = 12345", +[{rabbitmq_management, [{listener, [{port, 12345}]}]}], +[rabbitmq_management]} +, +{24, +"management.listener.port = 15671 +management.listener.ssl = true +management.listener.ssl_opts.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +management.listener.ssl_opts.certfile = test/config_schema_SUITE_data/certs/cert.pem +management.listener.ssl_opts.keyfile = test/config_schema_SUITE_data/certs/key.pem", +[{rabbitmq_management, + [{listener, [{port, 15671}, + {ssl, true}, + {ssl_opts, [{cacertfile, "test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}]} + ]} + ]} +], +[rabbitmq_management]} +, +{25, +"management.sample_retention_policies.global.minute = 5 +management.sample_retention_policies.global.hour = 60 +management.sample_retention_policies.global.day = 1200 + +management.sample_retention_policies.basic.minute = 5 +management.sample_retention_policies.basic.hour = 60 + +management.sample_retention_policies.detailed.10 = 5", +[{rabbitmq_management,[ + {sample_retention_policies, + %% List of {MaxAgeInSeconds, SampleEveryNSeconds} + [{global, [{60, 5}, {3600, 60}, {86400, 1200}]}, + {basic, [{60, 5}, {3600, 60}]}, + {detailed, [{10, 5}]}]} +]}], +[rabbitmq_management]} +, +{26, +"vm_memory_high_watermark.absolute = 1073741824", +[{rabbit, [{vm_memory_high_watermark, {absolute, 1073741824}}]}],[]} +, +{27, +"vm_memory_high_watermark.absolute = 1024MB", +[{rabbit, [{vm_memory_high_watermark, {absolute, "1024MB"}}]}],[]} +, +{28, +"vm_memory_high_watermark_paging_ratio = 0.75 +vm_memory_high_watermark.relative = 0.4", +[{rabbit, [{vm_memory_high_watermark_paging_ratio, 0.75}, + {vm_memory_high_watermark, 0.4}]}],[]} +, +{29, +"listeners.tcp.default = 5672 +mqtt.default_user = guest +mqtt.default_pass = guest +mqtt.allow_anonymous = true +mqtt.vhost = / +mqtt.exchange = amq.topic +mqtt.subscription_ttl = 1800000 +mqtt.prefetch = 10 +mqtt.listeners.ssl = none +## Default MQTT with TLS port is 8883 +# mqtt.listeners.ssl.default = 8883 +mqtt.listeners.tcp.default = 1883 +mqtt.tcp_listen_options.backlog = 128 +mqtt.tcp_listen_options.nodelay = true", +[{rabbit, [{tcp_listeners, [5672]}]}, + {rabbitmq_mqtt, [{default_user, <<"guest">>}, + {default_pass, <<"guest">>}, + {allow_anonymous, true}, + {vhost, <<"/">>}, + {exchange, <<"amq.topic">>}, + {subscription_ttl, 1800000}, + {prefetch, 10}, + {ssl_listeners, []}, + %% Default MQTT with TLS port is 8883 + %% {ssl_listeners, [8883]} + {tcp_listeners, [1883]}, + {tcp_listen_options, [{backlog, 128}, + {nodelay, true}]}]} + ], +[rabbitmq_mqtt]} +, +{30, +"ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true + +mqtt.listeners.ssl.default = 8883 +mqtt.listeners.tcp.default = 1883", +[{rabbit, [ + {ssl_options, [{cacertfile, "test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}]} + ]}, + {rabbitmq_mqtt, [ + {ssl_listeners, [8883]}, + {tcp_listeners, [1883]} + ]} + ], +[rabbitmq_mqtt]} +, +{31, +"mqtt.ssl_cert_login = true", +[{rabbitmq_mqtt, [{ssl_cert_login, true}]}], [rabbitmq_mqtt]} +, + +{32, +"ssl_cert_login_from = common_name", +[{rabbit, [{ssl_cert_login_from, common_name}]}], [rabbitmq_mqtt]} +, + + +{33, +"listeners.tcp.default = 5672 +mqtt.default_user = guest +mqtt.default_pass = guest +mqtt.allow_anonymous = true +mqtt.vhost = / +mqtt.exchange = amq.topic +mqtt.subscription_ttl = undefined +mqtt.prefetch = 10", +[{rabbit, [{tcp_listeners, [5672]}]}, + {rabbitmq_mqtt, [{default_user, <<"guest">>}, + {default_pass, <<"guest">>}, + {allow_anonymous, true}, + {vhost, <<"/">>}, + {exchange, <<"amq.topic">>}, + {subscription_ttl, undefined}, + {prefetch, 10}]} + ], +[rabbitmq_mqtt]} +, +{34, +"mqtt.default_user = guest +mqtt.default_pass = guest +mqtt.allow_anonymous = true +mqtt.vhost = / +mqtt.exchange = amq.topic +mqtt.subscription_ttl = 1800000 +mqtt.prefetch = 10 +## use DETS (disk-based) store for retained messages +mqtt.retained_message_store = rabbit_mqtt_retained_msg_store_dets +## only used by DETS store +mqtt.retained_message_store_dets_sync_interval = 2000 + +mqtt.listeners.ssl = none +mqtt.listeners.tcp.default = 1883", +[{rabbitmq_mqtt, [{default_user, <<"guest">>}, + {default_pass, <<"guest">>}, + {allow_anonymous, true}, + {vhost, <<"/">>}, + {exchange, <<"amq.topic">>}, + {subscription_ttl, 1800000}, + {prefetch, 10}, + %% use DETS (disk-based) store for retained messages + {retained_message_store, rabbit_mqtt_retained_msg_store_dets}, + %% only used by DETS store + {retained_message_store_dets_sync_interval, 2000}, + {ssl_listeners, []}, + {tcp_listeners, [1883]}]} + ], +[rabbitmq_mqtt]} +, + +{35, +"listeners.tcp.1 = 192.168.1.99:5672", +[ + {rabbit, [ + {tcp_listeners, [{"192.168.1.99", 5672}]} + ]} +], []} +, +{36, +"listeners.tcp.1 = 127.0.0.1:5672 +listeners.tcp.2 = ::1:5672", +[ + {rabbit, [ + {tcp_listeners, [{"127.0.0.1", 5672}, + {"::1", 5672}]} + ]} +], []} +, +{37, +"listeners.tcp.1 = :::5672", +[ + {rabbit, [ + {tcp_listeners, [{"::", 5672}]} + ]} +], []} +, +{38, +"listeners.tcp.1 = 192.168.1.99:5672", +[ + {rabbit, [ + {tcp_listeners, [{"192.168.1.99", 5672}]} + ]} +], []} +, +{39, +"listeners.tcp.1 = fe80::2acf:e9ff:fe17:f97b:5672", +[ + {rabbit, [ + {tcp_listeners, [{"fe80::2acf:e9ff:fe17:f97b", 5672}]} + ]} +], []} +, +{40, +"tcp_listen_options.backlog = 128 +tcp_listen_options.nodelay = true +tcp_listen_options.sndbuf = 196608 +tcp_listen_options.recbuf = 196608", +[ + {rabbit, [ + {tcp_listen_options, [ + {backlog, 128}, + {nodelay, true}, + {sndbuf, 196608}, + {recbuf, 196608} + ]} + ]} +], []} +, + +{42, +"tcp_listen_options.backlog = 4096 +tcp_listen_options.nodelay = true", +[ + {kernel, [ + {inet_default_connect_options, [{nodelay, true}]}, + {inet_default_listen_options, [{nodelay, true}]} + ]}] +, +[ + {kernel, [ + {inet_default_connect_options, [{nodelay, true}]}, + {inet_default_listen_options, [{nodelay, true}]} + ]}, + {rabbit, [ + {tcp_listen_options, [ + {backlog, 4096}, + {nodelay, true} + ]} + ]} +], []} +, + +{43, +"tcp_listen_options.backlog = 4096 +tcp_listen_options.nodelay = true", +[ + {rabbit, [ + {tcp_listen_options, [ + {backlog, 4096}, + {nodelay, true} + ]} + ]} +], []} +, + +{44, +"ssl_handshake_timeout = 10000", +[ + {rabbit, [ + %% 10 seconds + {ssl_handshake_timeout, 10000} + ]} +], []} +, + +{45, +"cluster_partition_handling = pause_if_all_down + +## Recover strategy. Can be either 'autoheal' or 'ignore' +cluster_partition_handling.pause_if_all_down.recover = ignore + +## Node names to check +cluster_partition_handling.pause_if_all_down.nodes.1 = rabbit@myhost1 +cluster_partition_handling.pause_if_all_down.nodes.2 = rabbit@myhost2", +[{rabbit, [{cluster_partition_handling, {pause_if_all_down, [rabbit@myhost2, rabbit@myhost1], ignore}}]}], []} +, +{46, +"cluster_partition_handling = autoheal", +[{rabbit, [{cluster_partition_handling, autoheal}]}], []} +, +{47, +"password_hashing_module = rabbit_password_hashing_sha512", +[ + {rabbit, [{password_hashing_module, rabbit_password_hashing_sha512}]} +],[]} +, + +{48, +"listeners.ssl.1 = 5671 +ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = false" +, +[ + {rabbit, [ + {ssl_listeners, [5671]}, + {ssl_options, [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]} + ]} +],[]} +, + + +{49, +"listeners.ssl.1 = 5671 +ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.password = t0p$3kRe7", +[ + {rabbit, [ + {ssl_listeners, [5671]}, + {ssl_options, [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {password, "t0p$3kRe7"} + ]} + ]} +],[]} +, + +{50, +"listeners.ssl.1 = 5671 +ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.versions.tls1_2 = tlsv1.2 +ssl_options.versions.tls1_1 = tlsv1.1 +ssl_options.versions.tls1 = tlsv1", +[{ssl, [{versions, ['tlsv1.2', 'tlsv1.1', tlsv1]}]}], +[{ssl, [{versions, ['tlsv1.2', 'tlsv1.1', tlsv1]}]}, + {rabbit, [ + {ssl_listeners, [5671]}, + {ssl_options, [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {versions, ['tlsv1.2', 'tlsv1.1', tlsv1]} + ]} + ]} +],[]} +, +{51, +"listeners.ssl.1 = 5671 +ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.versions.tls1_2 = tlsv1.2 +ssl_options.versions.tls1_1 = tlsv1.1", +[{ssl, [{versions, ['tlsv1.2', 'tlsv1.1']}]}], +[ + {ssl, [{versions, ['tlsv1.2', 'tlsv1.1']}]}, + {rabbit, [ + {ssl_listeners, [5671]}, + {ssl_options, [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {versions, ['tlsv1.2', 'tlsv1.1']} + ]} + ]} +],[]} +, +{52, +"listeners.ssl.1 = 5671 +ssl_allow_poodle_attack = true +ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = false", +[ + {rabbit, [ + {ssl_listeners, [5671]}, + {ssl_allow_poodle_attack, true}, + {ssl_options, [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]} + ]} +],[]} +, +{53, +"listeners.ssl.1 = 5671 +ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.depth = 2 +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = false", +[ + {rabbit, [ + {ssl_listeners, [5671]}, + {ssl_options, [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {depth, 2}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]} + ]} +],[]} +, +{54, +"stomp.listeners.tcp.1 = 12345", +[{rabbitmq_stomp, [{tcp_listeners, [12345]}]}],[rabbitmq_stomp]} +, +{55, +"stomp.listeners.tcp.1 = 127.0.0.1:61613 +stomp.listeners.tcp.2 = ::1:61613", +[{rabbitmq_stomp, [{tcp_listeners, [{"127.0.0.1", 61613}, + {"::1", 61613}]}]}],[rabbitmq_stomp]} +, +{56, +"ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true + +stomp.listeners.tcp.1 = 61613 +stomp.listeners.ssl.1 = 61614", +[{rabbit,[ +{ssl_options, [{cacertfile, "test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}]}]}, +{rabbitmq_stomp, [{tcp_listeners, [61613]}, +{ssl_listeners, [61614]}]} + ],[]} +, + +{57, +"stomp.default_user = guest +stomp.default_pass = guest", +[{rabbitmq_stomp, [{default_user, [{login, "guest"},{passcode, "guest"}]}]}], +[rabbitmq_stomp]} +, +{58, +"stomp.ssl_cert_login = true", +[{rabbitmq_stomp, [{ssl_cert_login, true}]}], +[rabbitmq_stomp]} +, +{59, +"ssl_cert_login_from = common_name", +[{rabbit, [{ssl_cert_login_from, common_name}]}], []} +, +{60, +"stomp.default_user = guest +stomp.default_pass = guest +stomp.implicit_connect = true", +[{rabbitmq_stomp, [{default_user,[{login, "guest"}, {passcode, "guest"}]},{implicit_connect, true}]}], +[rabbitmq_stomp]} +, +{61, +"stomp.default_vhost = /", +[{rabbitmq_stomp, [{default_vhost, <<"/">>}]}], +[rabbitmq_stomp]} +, +{62, +"management.listener.port = 15672 +management.listener.ip = 127.0.0.1", +[{rabbitmq_management, + [{listener, [{port, 15672}, + {ip, "127.0.0.1"} + ]} + ]} +], +[rabbitmq_management]} +, +{63, +"management.listener.port = 15672 +management.listener.ssl = true + +management.listener.ssl_opts.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +management.listener.ssl_opts.certfile = test/config_schema_SUITE_data/certs/cert.pem +management.listener.ssl_opts.keyfile = test/config_schema_SUITE_data/certs/key.pem", +[{rabbitmq_management, + [{listener, [{port, 15672}, + {ssl, true}, + {ssl_opts, [{cacertfile, "test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}]} + ]} + ]} +], +[rabbitmq_management]}, +{64, +"web_stomp.port = 12345", +[{rabbitmq_web_stomp, [{port, 12345}]}], +[rabbitmq_web_stomp]}, +{65, +"web_stomp.ssl.port = 15671 +web_stomp.ssl.backlog = 1024 +web_stomp.ssl.certfile = test/config_schema_SUITE_data/certs/cert.pem +web_stomp.ssl.keyfile = test/config_schema_SUITE_data/certs/key.pem +web_stomp.ssl.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +web_stomp.ssl.password = changeme", +[{rabbitmq_web_stomp, + [{ssl_config, [{port, 15671}, + {backlog, 1024}, + {certfile, "test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile, "test/config_schema_SUITE_data/certs/key.pem"}, + {cacertfile, "test/config_schema_SUITE_data/certs/cacert.pem"}, + {password, "changeme"}]}]}], +[rabbitmq_web_stomp]}, +{66, +"web_stomp.ws_frame = binary", +[{rabbitmq_web_stomp, [{ws_frame, binary}]}], +[rabbitmq_web_stomp]}, +{67, +"web_stomp.cowboy_opts.max_keepalive = 10", +[{rabbitmq_web_stomp,[{cowboy_opts, [{max_keepalive, 10}]}]}], +[rabbitmq_web_stomp]}, +{68, +"web_stomp.sockjs_opts.url = https://cdn.jsdelivr.net/sockjs/0.3.4/sockjs.min.js", +[{rabbitmq_web_stomp, + [{sockjs_opts, [{sockjs_url, "https://cdn.jsdelivr.net/sockjs/0.3.4/sockjs.min.js"}]}]}], +[rabbitmq_web_stomp]}, +{69, +"auth_backends.1 = http +rabbitmq_auth_backend_http.user_path = http://some-server/auth/user +rabbitmq_auth_backend_http.vhost_path = http://some-server/auth/vhost +rabbitmq_auth_backend_http.resource_path = http://some-server/auth/resource", +[{rabbit, [{auth_backends, [rabbit_auth_backend_http]}]}, + {rabbitmq_auth_backend_http, + [{user_path, "http://some-server/auth/user"}, + {vhost_path, "http://some-server/auth/vhost"}, + {resource_path, "http://some-server/auth/resource"}]}], +[rabbitmq_auth_backend_http]}, +{70, +"tcp_listen_options.linger.on = true +tcp_listen_options.linger.timeout = 100", +[{rabbit, [{tcp_listen_options, [{linger, {true, 100}}]}]}], +[]}, +{72, +"tcp_listen_options.linger.on = false +tcp_listen_options.linger.timeout = 100", +[{rabbit, [{tcp_listen_options, [{linger, {false, 100}}]}]}], +[]}, +{73, +"tcp_listen_options.linger.on = true", +[{rabbit, [{tcp_listen_options, [{linger, {true, 0}}]}]}], +[]}, +{74, +"tcp_listen_options.linger.timeout = 100", +[{rabbit, [{tcp_listen_options, [{linger, {false, 100}}]}]}], +[]} +]. diff --git a/test/dummy_interceptor.erl b/test/dummy_interceptor.erl new file mode 100644 index 0000000000..6d510a3073 --- /dev/null +++ b/test/dummy_interceptor.erl @@ -0,0 +1,26 @@ +-module(dummy_interceptor). + +-behaviour(rabbit_channel_interceptor). + +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + + +-compile(export_all). + +init(_Ch) -> + undefined. + +description() -> + [{description, + <<"Empties payload on publish">>}]. + +intercept(#'basic.publish'{} = Method, Content, _IState) -> + Content2 = Content#content{payload_fragments_rev = []}, + {Method, Content2}; + +intercept(Method, Content, _VHost) -> + {Method, Content}. + +applies_to() -> + ['basic.publish']. diff --git a/test/per_vhost_connection_limit_SUITE.erl b/test/per_vhost_connection_limit_SUITE.erl new file mode 100644 index 0000000000..592e57c41a --- /dev/null +++ b/test/per_vhost_connection_limit_SUITE.erl @@ -0,0 +1,815 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2011-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(per_vhost_connection_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_1_direct}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + most_basic_single_node_connection_count, + single_node_single_vhost_connection_count, + single_node_multiple_vhosts_connection_count, + single_node_list_in_vhost, + single_node_single_vhost_limit, + single_node_single_vhost_zero_limit, + single_node_multiple_vhosts_limit, + single_node_multiple_vhosts_zero_limit, + single_node_vhost_deletion_forces_connection_closure + ], + ClusterSize2Tests = [ + most_basic_cluster_connection_count, + cluster_single_vhost_connection_count, + cluster_multiple_vhosts_connection_count, + cluster_node_restart_connection_count, + cluster_node_list_on_node, + cluster_single_vhost_limit, + cluster_single_vhost_limit2, + cluster_single_vhost_zero_limit, + cluster_multiple_vhosts_zero_limit, + cluster_vhost_deletion_forces_connection_closure + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_1_direct, [], ClusterSize1Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests}, + {cluster_rename, [], [ + vhost_limit_after_node_renamed + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 9000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config, [ + fun rabbit_ct_broker_helpers:enable_dist_proxy_manager/1 + ]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_1_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_1_direct, Config1, 1); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2); + +init_per_group(cluster_rename, Config) -> + init_per_multinode_group(cluster_rename, Config, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + case Group of + cluster_rename -> + % The broker is managed by {init,end}_per_testcase(). + Config1; + _ -> + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()) + end. + +end_per_group(cluster_rename, Config) -> + % The broker is managed by {init,end}_per_testcase(). + Config; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + Config. + +end_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + Config1 = ?config(save_config, Config), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase); +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracked_connection_tables_for_this_node, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +most_basic_single_node_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + [Conn] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +single_node_single_vhost_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn4] = open_connections(Config, [0]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + kill_connections([Conn4]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn5] = open_connections(Config, [0]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + close_connections([Conn2, Conn3, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +single_node_multiple_vhosts_connection_count(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + + [Conn2] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn3] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn4] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(2, count_connections_in(Config, VHost1)), + + kill_connections([Conn4]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [Conn5] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(2, count_connections_in(Config, VHost2)), + + [Conn6] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(3, count_connections_in(Config, VHost2)), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +single_node_list_in_vhost(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, length(connections_in(Config, VHost1))), + ?assertEqual(0, length(connections_in(Config, VHost2))), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + [#tracked_connection{vhost = VHost1}] = connections_in(Config, VHost1), + close_connections([Conn1]), + ?assertEqual(0, length(connections_in(Config, VHost1))), + + [Conn2] = open_connections(Config, [{0, VHost2}]), + [#tracked_connection{vhost = VHost2}] = connections_in(Config, VHost2), + + [Conn3] = open_connections(Config, [{0, VHost1}]), + [#tracked_connection{vhost = VHost1}] = connections_in(Config, VHost1), + + [Conn4] = open_connections(Config, [{0, VHost1}]), + kill_connections([Conn4]), + [#tracked_connection{vhost = VHost1}] = connections_in(Config, VHost1), + + [Conn5, Conn6] = open_connections(Config, [{0, VHost2}, {0, VHost2}]), + [<<"vhost1">>, <<"vhost2">>] = + lists:usort(lists:map(fun (#tracked_connection{vhost = V}) -> V end, + all_connections(Config))), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, length(all_connections(Config))), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +most_basic_cluster_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [1]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + close_connections([Conn1, Conn2, Conn3]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +cluster_single_vhost_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [1]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn4] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + kill_connections([Conn4]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn5] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + close_connections([Conn2, Conn3, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +cluster_multiple_vhosts_connection_count(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + + [Conn2] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn3] = open_connections(Config, [{1, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn4] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(2, count_connections_in(Config, VHost1)), + + kill_connections([Conn4]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [Conn5] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(2, count_connections_in(Config, VHost2)), + + [Conn6] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(3, count_connections_in(Config, VHost2)), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +cluster_node_restart_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [1]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn4] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + [Conn5] = open_connections(Config, [1]), + ?assertEqual(4, count_connections_in(Config, VHost)), + + rabbit_ct_broker_helpers:restart_broker(Config, 1), + ?assertEqual(1, count_connections_in(Config, VHost)), + + close_connections([Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +cluster_node_list_on_node(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + ?assertEqual(0, length(all_connections(Config))), + ?assertEqual(0, length(connections_on_node(Config, 0))), + + [Conn1] = open_connections(Config, [0]), + [#tracked_connection{node = A}] = connections_on_node(Config, 0), + close_connections([Conn1]), + ?assertEqual(0, length(connections_on_node(Config, 0))), + + [_Conn2] = open_connections(Config, [1]), + [#tracked_connection{node = B}] = connections_on_node(Config, 1), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(1, length(connections_on_node(Config, 0))), + + [Conn4] = open_connections(Config, [1]), + ?assertEqual(2, length(connections_on_node(Config, 1))), + + kill_connections([Conn4]), + ?assertEqual(1, length(connections_on_node(Config, 1))), + + [Conn5] = open_connections(Config, [0]), + ?assertEqual(2, length(connections_on_node(Config, 0))), + + rabbit_ct_broker_helpers:stop_broker(Config, 1), + await_running_node_refresh(Config, 0), + + ?assertEqual(2, length(all_connections(Config))), + ?assertEqual(0, length(connections_on_node(Config, 0, B))), + + close_connections([Conn3, Conn5]), + ?assertEqual(0, length(all_connections(Config, 0))), + + rabbit_ct_broker_helpers:start_broker(Config, 1). + +single_node_single_vhost_limit(Config) -> + single_node_single_vhost_limit_with(Config, 5), + single_node_single_vhost_limit_with(Config, -1). + +single_node_single_vhost_limit_with(Config, WatermarkLimit) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 3), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1, Conn2, Conn3] = open_connections(Config, [0, 0, 0]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + + set_vhost_connection_limit(Config, VHost, WatermarkLimit), + [Conn4, Conn5] = open_connections(Config, [0, 0]), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + +single_node_single_vhost_zero_limit(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 0), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + + set_vhost_connection_limit(Config, VHost, -1), + [Conn1, Conn2] = open_connections(Config, [0, 0]), + + close_connections([Conn1, Conn2]), + ?assertEqual(0, count_connections_in(Config, VHost)). + + +single_node_multiple_vhosts_limit(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + set_vhost_connection_limit(Config, VHost1, 2), + set_vhost_connection_limit(Config, VHost2, 2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1, Conn2, Conn3, Conn4] = open_connections(Config, [ + {0, VHost1}, + {0, VHost1}, + {0, VHost2}, + {0, VHost2}]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, VHost1), + expect_that_client_connection_is_rejected(Config, 0, VHost2), + + [Conn5] = open_connections(Config, [0]), + + set_vhost_connection_limit(Config, VHost1, 5), + set_vhost_connection_limit(Config, VHost2, -10), + + [Conn6, Conn7, Conn8, Conn9, Conn10] = open_connections(Config, [ + {0, VHost1}, + {0, VHost1}, + {0, VHost1}, + {0, VHost2}, + {0, VHost2}]), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5, + Conn6, Conn7, Conn8, Conn9, Conn10]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + + +single_node_multiple_vhosts_zero_limit(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + set_vhost_connection_limit(Config, VHost1, 0), + set_vhost_connection_limit(Config, VHost2, 0), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, VHost1), + expect_that_client_connection_is_rejected(Config, 0, VHost2), + expect_that_client_connection_is_rejected(Config, 0, VHost1), + + set_vhost_connection_limit(Config, VHost1, -1), + [Conn1, Conn2] = open_connections(Config, [{0, VHost1}, {0, VHost1}]), + + close_connections([Conn1, Conn2]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1). + + +cluster_single_vhost_limit(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 2), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% here connections are opened to different nodes + [Conn1, Conn2] = open_connections(Config, [{0, VHost}, {1, VHost}]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, VHost), + expect_that_client_connection_is_rejected(Config, 1, VHost), + + set_vhost_connection_limit(Config, VHost, 5), + + [Conn3, Conn4] = open_connections(Config, [{0, VHost}, {0, VHost}]), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + +cluster_single_vhost_limit2(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 2), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% here a limit is reached on one node first + [Conn1, Conn2] = open_connections(Config, [{0, VHost}, {0, VHost}]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, VHost), + expect_that_client_connection_is_rejected(Config, 1, VHost), + + set_vhost_connection_limit(Config, VHost, 5), + + [Conn3, Conn4, Conn5, {error, not_allowed}] = open_connections(Config, [ + {1, VHost}, + {1, VHost}, + {1, VHost}, + {1, VHost}]), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + + +cluster_single_vhost_zero_limit(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 0), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 1), + expect_that_client_connection_is_rejected(Config, 0), + + set_vhost_connection_limit(Config, VHost, -1), + [Conn1, Conn2, Conn3, Conn4] = open_connections(Config, [0, 1, 0, 1]), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + + +cluster_multiple_vhosts_zero_limit(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + set_vhost_connection_limit(Config, VHost1, 0), + set_vhost_connection_limit(Config, VHost2, 0), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, VHost1), + expect_that_client_connection_is_rejected(Config, 0, VHost2), + expect_that_client_connection_is_rejected(Config, 1, VHost1), + expect_that_client_connection_is_rejected(Config, 1, VHost2), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1), + + [Conn1, Conn2, Conn3, Conn4] = open_connections(Config, [ + {0, VHost1}, + {0, VHost2}, + {1, VHost1}, + {1, VHost2}]), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1). + + +single_node_vhost_deletion_forces_connection_closure(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [_Conn2] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1). + +cluster_vhost_deletion_forces_connection_closure(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [_Conn2] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1). + +vhost_limit_after_node_renamed(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + VHost = <<"/renaming_node">>, + set_up_vhost(Config, VHost), + set_vhost_connection_limit(Config, VHost, 2), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1, Conn2, {error, not_allowed}] = open_connections(Config, + [{0, VHost}, {1, VHost}, {0, VHost}]), + ?assertEqual(2, count_connections_in(Config, VHost)), + close_connections([Conn1, Conn2]), + + Config1 = cluster_rename_SUITE:stop_rename_start(Config, A, [A, 'new-A']), + + ?assertEqual(0, count_connections_in(Config1, VHost)), + + [Conn3, Conn4, {error, not_allowed}] = open_connections(Config, + [{0, VHost}, {1, VHost}, {0, VHost}]), + ?assertEqual(2, count_connections_in(Config1, VHost)), + close_connections([Conn3, Conn4]), + + set_vhost_connection_limit(Config1, VHost, -1), + {save_config, Config1}. + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndVHosts) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, VHost}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + VHost); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndVHosts), + timer:sleep(500), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns), + timer:sleep(500). + +kill_connections(Conns) -> + lists:foreach(fun + (Conn) -> + (catch exit(Conn, please_terminate)) + end, Conns), + timer:sleep(500). + +count_connections_in(Config, VHost) -> + count_connections_in(Config, VHost, 0). +count_connections_in(Config, VHost, NodeIndex) -> + timer:sleep(200), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + count_connections_in, [VHost]). + +connections_in(Config, VHost) -> + connections_in(Config, 0, VHost). +connections_in(Config, NodeIndex, VHost) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, [VHost]). + +connections_on_node(Config) -> + connections_on_node(Config, 0). +connections_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_on_node, [Node]). +connections_on_node(Config, NodeIndex, NodeForListing) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_on_node, [NodeForListing]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, []). + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_connection_limit(Config, VHost, -1). + +set_vhost_connection_limit(Config, VHost, Count) -> + set_vhost_connection_limit(Config, 0, VHost, Count). + +set_vhost_connection_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-connections\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, VHost) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, VHost). diff --git a/test/per_vhost_connection_limit_partitions_SUITE.erl b/test/per_vhost_connection_limit_partitions_SUITE.erl new file mode 100644 index 0000000000..051f3f13b7 --- /dev/null +++ b/test/per_vhost_connection_limit_partitions_SUITE.erl @@ -0,0 +1,170 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2011-2015 Pivotal Software, Inc. All rights reserved. +%% + +-module(per_vhost_connection_limit_partitions_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(rabbit_ct_client_helpers, [open_unmanaged_connection/2, + open_unmanaged_connection/3]). + + +all() -> + [ + {group, net_ticktime_1} + ]. + +groups() -> + [ + {net_ticktime_1, [], [ + cluster_full_partition_with_autoheal + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 12000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config, [ + fun rabbit_ct_broker_helpers:enable_dist_proxy_manager/1 + ]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(net_ticktime_1 = Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{net_ticktime, 1}]), + init_per_multinode_group(Group, Config1, 3). + +init_per_multinode_group(_Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix}, + {rmq_nodes_clustered, false} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun rabbit_ct_broker_helpers:enable_dist_proxy/1, + fun rabbit_ct_broker_helpers:cluster_nodes/1 + ]). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +cluster_full_partition_with_autoheal(Config) -> + VHost = <<"/">>, + rabbit_ct_broker_helpers:set_partition_handling_mode_globally(Config, autoheal), + + ?assertEqual(0, count_connections_in(Config, VHost)), + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% 6 connections, 2 per node + Conn1 = open_unmanaged_connection(Config, A), + Conn2 = open_unmanaged_connection(Config, A), + Conn3 = open_unmanaged_connection(Config, B), + Conn4 = open_unmanaged_connection(Config, B), + Conn5 = open_unmanaged_connection(Config, C), + Conn6 = open_unmanaged_connection(Config, C), + ?assertEqual(6, count_connections_in(Config, VHost)), + + %% B drops off the network, non-reachable by either A or C + rabbit_ct_broker_helpers:block_traffic_between(A, B), + rabbit_ct_broker_helpers:block_traffic_between(B, C), + timer:sleep(?DELAY), + + %% A and C are still connected, so 4 connections are tracked + ?assertEqual(4, count_connections_in(Config, VHost)), + + rabbit_ct_broker_helpers:allow_traffic_between(A, B), + rabbit_ct_broker_helpers:allow_traffic_between(B, C), + timer:sleep(?DELAY), + + %% during autoheal B's connections were dropped + ?assertEqual(4, count_connections_in(Config, VHost)), + + lists:foreach(fun (Conn) -> + (catch rabbit_ct_client_helpers:close_connection(Conn)) + end, [Conn1, Conn2, Conn3, Conn4, + Conn5, Conn6]), + + passed. + + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +count_connections_in(Config, VHost) -> + count_connections_in(Config, VHost, 0). +count_connections_in(Config, VHost, NodeIndex) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + count_connections_in, [VHost]). + +connections_in(Config, VHost) -> + connections_in(Config, 0, VHost). +connections_in(Config, NodeIndex, VHost) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, [VHost]). + +connections_on_node(Config) -> + connections_on_node(Config, 0). +connections_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_on_node, [Node]). +connections_on_node(Config, NodeIndex, NodeForListing) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_on_node, [NodeForListing]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, []). diff --git a/test/per_vhost_queue_limit_SUITE.erl b/test/per_vhost_queue_limit_SUITE.erl new file mode 100644 index 0000000000..1c6bea08dd --- /dev/null +++ b/test/per_vhost_queue_limit_SUITE.erl @@ -0,0 +1,693 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2011-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(per_vhost_queue_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(rabbit_ct_client_helpers, [open_unmanaged_connection/3, + close_connection_and_channel/2]). + +all() -> + [ + {group, cluster_size_1} + , {group, cluster_size_2} + ]. + +groups() -> + [ + {cluster_size_1, [], [ + most_basic_single_node_queue_count, + single_node_single_vhost_queue_count, + single_node_multiple_vhosts_queue_count, + single_node_single_vhost_limit, + single_node_single_vhost_zero_limit, + single_node_single_vhost_limit_with_durable_named_queue, + single_node_single_vhost_zero_limit_with_durable_named_queue, + single_node_single_vhost_limit_with_queue_ttl, + single_node_single_vhost_limit_with_redeclaration + ]}, + {cluster_size_2, [], [ + most_basic_cluster_queue_count, + cluster_multiple_vhosts_queue_count, + cluster_multiple_vhosts_limit, + cluster_multiple_vhosts_zero_limit, + cluster_multiple_vhosts_limit_with_durable_named_queue, + cluster_multiple_vhosts_zero_limit_with_durable_named_queue, + cluster_node_restart_queue_count + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config, [ + fun rabbit_ct_broker_helpers:enable_dist_proxy_manager/1 + ]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1, Config) -> + init_per_multinode_group(cluster_size_1, Config, 1); +init_per_group(cluster_size_2, Config) -> + init_per_multinode_group(cluster_size_2, Config, 2); +init_per_group(cluster_rename, Config) -> + init_per_multinode_group(cluster_rename, Config, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + case Group of + cluster_rename -> + % The broker is managed by {init,end}_per_testcase(). + Config1; + _ -> + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()) + end. + +end_per_group(cluster_rename, Config) -> + % The broker is managed by {init,end}_per_testcase(). + Config; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config. + +end_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + Config1 = ?config(save_config, Config), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase); +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +most_basic_single_node_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + declare_exclusive_queues(Ch, 10), + ?assertEqual(10, count_queues_in(Config, VHost)), + close_connection_and_channel(Conn, Ch), + ?assertEqual(0, count_queues_in(Config, VHost)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_single_vhost_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + declare_exclusive_queues(Ch, 10), + ?assertEqual(10, count_queues_in(Config, VHost)), + declare_durable_queues(Ch, 10), + ?assertEqual(20, count_queues_in(Config, VHost)), + delete_durable_queues(Ch, 10), + ?assertEqual(10, count_queues_in(Config, VHost)), + close_connection_and_channel(Conn, Ch), + ?assertEqual(0, count_queues_in(Config, VHost)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_multiple_vhosts_queue_count(Config) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + Conn2 = open_unmanaged_connection(Config, 0, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost1)), + declare_durable_queues(Ch1, 10), + ?assertEqual(20, count_queues_in(Config, VHost1)), + delete_durable_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost1)), + declare_exclusive_queues(Ch2, 30), + ?assertEqual(30, count_queues_in(Config, VHost2)), + close_connection_and_channel(Conn1, Ch1), + ?assertEqual(0, count_queues_in(Config, VHost1)), + close_connection_and_channel(Conn2, Ch2), + ?assertEqual(0, count_queues_in(Config, VHost2)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +single_node_single_vhost_limit(Config) -> + single_node_single_vhost_limit_with(Config, 5), + single_node_single_vhost_limit_with(Config, 10). +single_node_single_vhost_zero_limit(Config) -> + single_node_single_vhost_zero_limit_with(Config, #'queue.declare'{queue = <<"">>, + exclusive = true}). + +single_node_single_vhost_limit_with_durable_named_queue(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + set_vhost_queue_limit(Config, VHost, 3), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_single_vhost_zero_limit_with_durable_named_queue(Config) -> + single_node_single_vhost_zero_limit_with(Config, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}). + +single_node_single_vhost_limit_with(Config, WatermarkLimit) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + set_vhost_queue_limit(Config, VHost, 3), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + + set_vhost_queue_limit(Config, VHost, WatermarkLimit), + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, WatermarkLimit)), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_single_vhost_zero_limit_with(Config, QueueDeclare) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + set_vhost_queue_limit(Config, VHost, 0), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, QueueDeclare) + end), + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + %% lift the limit + set_vhost_queue_limit(Config, VHost, -1), + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, 100)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +single_node_single_vhost_limit_with_queue_ttl(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + set_vhost_queue_limit(Config, VHost, 3), + + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true, + arguments = [{<<"x-expires">>, long, 2000}]}) + end, lists:seq(1, 3)), + + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + %% wait for the queues to expire + timer:sleep(3000), + + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +single_node_single_vhost_limit_with_redeclaration(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + set_vhost_queue_limit(Config, VHost, 3), + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + %% can't declare a new queue... + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}) + end), + + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + %% ...but re-declarations succeed + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +most_basic_cluster_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + ?assertEqual(0, count_queues_in(Config, VHost, 1)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost, 0)), + ?assertEqual(10, count_queues_in(Config, VHost, 1)), + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + declare_exclusive_queues(Ch2, 15), + ?assertEqual(25, count_queues_in(Config, VHost, 0)), + ?assertEqual(25, count_queues_in(Config, VHost, 1)), + close_connection_and_channel(Conn1, Ch1), + close_connection_and_channel(Conn2, Ch2), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + ?assertEqual(0, count_queues_in(Config, VHost, 1)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +cluster_node_restart_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + ?assertEqual(0, count_queues_in(Config, VHost, 1)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost, 0)), + ?assertEqual(10, count_queues_in(Config, VHost, 1)), + + rabbit_ct_broker_helpers:restart_broker(Config, 0), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + + Conn2 = open_unmanaged_connection(Config, 1, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + declare_exclusive_queues(Ch2, 15), + ?assertEqual(15, count_queues_in(Config, VHost, 0)), + ?assertEqual(15, count_queues_in(Config, VHost, 1)), + + declare_durable_queues(Ch2, 10), + ?assertEqual(25, count_queues_in(Config, VHost, 0)), + ?assertEqual(25, count_queues_in(Config, VHost, 1)), + + rabbit_ct_broker_helpers:restart_broker(Config, 1), + + ?assertEqual(10, count_queues_in(Config, VHost, 0)), + ?assertEqual(10, count_queues_in(Config, VHost, 1)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +cluster_multiple_vhosts_queue_count(Config) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost1, 0)), + ?assertEqual(10, count_queues_in(Config, VHost1, 1)), + ?assertEqual(0, count_queues_in(Config, VHost2, 0)), + ?assertEqual(0, count_queues_in(Config, VHost2, 1)), + + Conn2 = open_unmanaged_connection(Config, 0, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + declare_exclusive_queues(Ch2, 15), + ?assertEqual(15, count_queues_in(Config, VHost2, 0)), + ?assertEqual(15, count_queues_in(Config, VHost2, 1)), + close_connection_and_channel(Conn1, Ch1), + close_connection_and_channel(Conn2, Ch2), + ?assertEqual(0, count_queues_in(Config, VHost1, 0)), + ?assertEqual(0, count_queues_in(Config, VHost1, 1)), + ?assertEqual(0, count_queues_in(Config, VHost2, 0)), + ?assertEqual(0, count_queues_in(Config, VHost2, 1)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +cluster_multiple_vhosts_limit(Config) -> + cluster_multiple_vhosts_limit_with(Config, 10), + cluster_multiple_vhosts_limit_with(Config, 20). + +cluster_multiple_vhosts_limit_with(Config, WatermarkLimit) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + set_vhost_queue_limit(Config, VHost1, 3), + set_vhost_queue_limit(Config, VHost2, 3), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + set_vhost_queue_limit(Config, VHost1, WatermarkLimit), + + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, WatermarkLimit)), + + Conn2 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + set_vhost_queue_limit(Config, VHost2, WatermarkLimit), + + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, WatermarkLimit)), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + + +cluster_multiple_vhosts_zero_limit(Config) -> + cluster_multiple_vhosts_zero_limit_with(Config, #'queue.declare'{queue = <<"">>, + exclusive = true}). + +cluster_multiple_vhosts_limit_with_durable_named_queue(Config) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + set_vhost_queue_limit(Config, VHost1, 3), + set_vhost_queue_limit(Config, VHost2, 3), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + Conn2 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + #'queue.declare_ok'{} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + #'queue.declare_ok'{} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}) + end), + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +cluster_multiple_vhosts_zero_limit_with_durable_named_queue(Config) -> + cluster_multiple_vhosts_zero_limit_with(Config, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}). + +cluster_multiple_vhosts_zero_limit_with(Config, QueueDeclare) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + Conn2 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + set_vhost_queue_limit(Config, VHost1, 0), + set_vhost_queue_limit(Config, VHost2, 0), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, QueueDeclare) + end), + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, QueueDeclare) + end), + + + Conn3 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch3} = amqp_connection:open_channel(Conn3), + Conn4 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch4} = amqp_connection:open_channel(Conn4), + + %% lift the limits + set_vhost_queue_limit(Config, VHost1, -1), + set_vhost_queue_limit(Config, VHost2, -1), + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch3, #'queue.declare'{queue = <<"">>, + exclusive = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch4, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, 400)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + + + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_queue_limit(Config, VHost, -1). + +set_vhost_queue_limit(Config, VHost, Count) -> + set_vhost_queue_limit(Config, 0, VHost, Count). + +set_vhost_queue_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-queues\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +count_queues_in(Config, VHost) -> + count_queues_in(Config, VHost, 0). +count_queues_in(Config, VHost, NodeIndex) -> + timer:sleep(200), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_amqqueue, + count, [VHost]). + +declare_exclusive_queues(Ch, N) -> + lists:foreach(fun (_) -> + amqp_channel:call(Ch, + #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, + lists:seq(1, N)). + +declare_durable_queues(Ch, N) -> + lists:foreach(fun (I) -> + amqp_channel:call(Ch, + #'queue.declare'{queue = durable_queue_name(I), + exclusive = false, + durable = true}) + end, + lists:seq(1, N)). + +delete_durable_queues(Ch, N) -> + lists:foreach(fun (I) -> + amqp_channel:call(Ch, + #'queue.delete'{queue = durable_queue_name(I)}) + end, + lists:seq(1, N)). + +durable_queue_name(N) when is_integer(N) -> + iolist_to_binary(io_lib:format("queue-limits-durable-~p", [N])). + +expect_shutdown_due_to_precondition_failed(Thunk) -> + try + Thunk(), + ok + catch _:{{shutdown, {server_initiated_close, 406, _}}, _} -> + %% expected + ok + end. diff --git a/test/plugin_versioning_SUITE.erl b/test/plugin_versioning_SUITE.erl new file mode 100644 index 0000000000..9c000557b8 --- /dev/null +++ b/test/plugin_versioning_SUITE.erl @@ -0,0 +1,177 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(plugin_versioning_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + version_support, + plugin_validation + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +version_support(_Config) -> + Examples = [ + {[], "any version", true} %% anything goes + ,{[], "0.0.0", true} %% ditto + ,{[], "3.5.6", true} %% ditto + ,{["something"], "something", true} %% equal values match + ,{["3.5.4"], "something", false} + ,{["3.4.5", "3.6.0"], "0.0.0", true} %% zero version always match + ,{["3.4.5", "3.6.0"], "", true} %% empty version always match + ,{["something", "3.5.6"], "3.5.7", true} %% 3.5.7 matches ~> 3.5.6 + ,{["3.4.0", "3.5.6"], "3.6.1", false} %% 3.6.x isn't supported + ,{["3.5.2", "3.6.1", "3.7.1"], "3.5.2", true} %% 3.5.2 matches ~> 3.5.2 + ,{["3.5.2", "3.6.1", "3.7.1"], "3.5.1", false} %% lesser than the lower boundary + ,{["3.5.2", "3.6.1", "3.7.1"], "3.6.2", true} %% 3.6.2 matches ~> 3.6.1 + ,{["3.5.2", "3.6.1", "3.6.8"], "3.6.2", true} %% 3.6.2 still matches ~> 3.6.1 + ,{["3.5", "3.6", "3.7"], "3.5.1", true} %% x.y values equal to x.y.0 + ,{["3"], "3.5.1", false} %% x values are not supported + ,{["3.5.2", "3.6.1"], "3.6.2.999", true} %% x.y.z.p values are supported + ,{["3.5.2", "3.6.2.333"], "3.6.2.999", true} %% x.y.z.p values are supported + ,{["3.5.2", "3.6.2.333"], "3.6.2.222", false} %% x.y.z.p values are supported + ], + + lists:foreach( + fun({Versions, RabbitVersion, Expected}) -> + {Expected, RabbitVersion, Versions} = + {rabbit_plugins:is_version_supported(RabbitVersion, Versions), + RabbitVersion, Versions} + end, + Examples), + ok. + +-record(validation_example, {rabbit_version, plugins, errors, valid}). + +plugin_validation(_Config) -> + Examples = [ + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.2", ["3.5.6", "3.7.1"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.6.3", "3.7.1"]}]}], + errors = [], + valid = [plugin_a, plugin_b]}, + + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.1", ["3.7.6"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.6.3", "3.7.0"]}]}], + errors = + [{plugin_a, [{broker_version_mismatch, "3.7.1", ["3.7.6"]}]}, + {plugin_b, [{missing_dependency, plugin_a}]}], + valid = [] + }, + + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.1", ["3.7.6"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.7.0"]}]}, + {plugin_c, "3.7.2", ["3.7.0"], [{plugin_b, ["3.7.3"]}]}], + errors = + [{plugin_a, [{broker_version_mismatch, "3.7.1", ["3.7.6"]}]}, + {plugin_b, [{missing_dependency, plugin_a}]}, + {plugin_c, [{missing_dependency, plugin_b}]}], + valid = [] + }, + + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.1", ["3.7.1"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.7.3"]}]}, + {plugin_d, "3.7.2", ["3.7.0"], [{plugin_c, ["3.7.3"]}]}], + errors = + [{plugin_b, [{{dependency_version_mismatch, "3.7.1", ["3.7.3"]}, plugin_a}]}, + {plugin_d, [{missing_dependency, plugin_c}]}], + valid = [plugin_a] + }, + #validation_example{ + rabbit_version = "0.0.0", + plugins = + [{plugin_a, "", ["3.7.1"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.7.3"]}]}], + errors = [], + valid = [plugin_a, plugin_b] + }], + lists:foreach( + fun(#validation_example{rabbit_version = RabbitVersion, + plugins = PluginsExamples, + errors = Errors, + valid = ExpectedValid}) -> + Plugins = make_plugins(PluginsExamples), + {Valid, Invalid} = rabbit_plugins:validate_plugins(Plugins, + RabbitVersion), + Errors = lists:reverse(Invalid), + ExpectedValid = lists:reverse(lists:map(fun(#plugin{name = Name}) -> + Name + end, + Valid)) + end, + Examples), + ok. + +make_plugins(Plugins) -> + lists:map( + fun({Name, Version, RabbitVersions, PluginsVersions}) -> + Deps = [K || {K,_V} <- PluginsVersions], + #plugin{name = Name, + version = Version, + dependencies = Deps, + broker_version_requirements = RabbitVersions, + dependency_version_requirements = PluginsVersions} + end, + Plugins). diff --git a/test/policy_SUITE.erl b/test/policy_SUITE.erl new file mode 100644 index 0000000000..634f198858 --- /dev/null +++ b/test/policy_SUITE.erl @@ -0,0 +1,157 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2011-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(policy_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_2} + ]. + +groups() -> + [ + {cluster_size_2, [], [ + policy_ttl, + operator_policy_ttl + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_client_helpers:setup_steps(), + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_client_helpers:teardown_steps(), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +policy_ttl(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"policy_ttl-queue">>, + rabbit_ct_broker_helpers:set_policy(Config, 0, <<"ttl-policy">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 20}]), + + declare(Ch, Q), + publish(Ch, Q, lists:seq(1, 20)), + timer:sleep(50), + get_empty(Ch, Q), + delete(Ch, Q), + + rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ttl-policy">>), + + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +operator_policy_ttl(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"policy_ttl-queue">>, + % Operator policy will override + rabbit_ct_broker_helpers:set_policy(Config, 0, <<"ttl-policy">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 100000}]), + rabbit_ct_broker_helpers:set_operator_policy(Config, 0, <<"ttl-policy-op">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 1}]), + + declare(Ch, Q), + publish(Ch, Q, lists:seq(1, 50)), + timer:sleep(50), + get_empty(Ch, Q), + delete(Ch, Q), + + rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ttl-policy">>), + rabbit_ct_broker_helpers:clear_operator_policy(Config, 0, <<"ttl-policy-op">>), + + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +%%---------------------------------------------------------------------------- + + +declare(Ch, Q) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true}). + +delete(Ch, Q) -> + amqp_channel:call(Ch, #'queue.delete'{queue = Q}). + +publish(Ch, Q, Ps) -> + amqp_channel:call(Ch, #'confirm.select'{}), + [publish1(Ch, Q, P) || P <- Ps], + amqp_channel:wait_for_confirms(Ch). + +publish1(Ch, Q, P) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = erlang:md5(term_to_binary(P))}). + +publish1(Ch, Q, P, Pd) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = Pd}). + +props(undefined) -> #'P_basic'{delivery_mode = 2}; +props(P) -> #'P_basic'{priority = P, + delivery_mode = 2}. + +consume(Ch, Q, Ack) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, + no_ack = Ack =:= no_ack, + consumer_tag = <<"ctag">>}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end. + +get_empty(Ch, Q) -> + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = Q}). + +%%---------------------------------------------------------------------------- diff --git a/test/priority_queue_SUITE.erl b/test/priority_queue_SUITE.erl index 05853ebc1f..8e1b48dd3b 100644 --- a/test/priority_queue_SUITE.erl +++ b/test/priority_queue_SUITE.erl @@ -49,7 +49,8 @@ groups() -> {cluster_size_3, [], [ mirror_queue_auto_ack, mirror_fast_reset_policy, - mirror_reset_policy + mirror_reset_policy, + mirror_stop_pending_slaves ]} ]. @@ -523,6 +524,36 @@ mirror_reset_policy(Config, Wait) -> rabbit_ct_client_helpers:close_connection(Conn), passed. +mirror_stop_pending_slaves(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + B = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + C = rabbit_ct_broker_helpers:get_node_config(Config, 2, nodename), + + [ok = rabbit_ct_broker_helpers:rpc( + Config, Nodename, application, set_env, [rabbit, slave_wait_timeout, 0]) || Nodename <- [A, B, C]], + + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"mirror_stop_pending_slaves-queue">>, + declare(Ch, Q, 5), + publish_many(Ch, Q, 20000), + + [begin + rabbit_ct_broker_helpers:set_ha_policy( + Config, A, <<"^mirror_stop_pending_slaves-queue$">>, <<"all">>, + [{<<"ha-sync-mode">>, <<"automatic">>}]), + wait_for_sync(Config, A, rabbit_misc:r(<<"/">>, queue, Q), 2), + rabbit_ct_broker_helpers:clear_policy( + Config, A, <<"^mirror_stop_pending_slaves-queue$">>) + end || _ <- lists:seq(1, 15)], + + delete(Ch, Q), + + [ok = rabbit_ct_broker_helpers:rpc( + Config, Nodename, application, set_env, [rabbit, slave_wait_timeout, 15000]) || Nodename <- [A, B, C]], + + rabbit_ct_client_helpers:close_connection(Conn), + passed. + %%---------------------------------------------------------------------------- declare(Ch, Q, Args) when is_list(Args) -> @@ -546,7 +577,7 @@ publish_payload(Ch, Q, PPds) -> amqp_channel:wait_for_confirms(Ch). publish_many(_Ch, _Q, 0) -> ok; -publish_many( Ch, Q, N) -> publish1(Ch, Q, rand_compat:uniform(5)), +publish_many( Ch, Q, N) -> publish1(Ch, Q, rand:uniform(5)), publish_many(Ch, Q, N - 1). publish1(Ch, Q, P) -> diff --git a/test/unit_SUITE.erl b/test/unit_SUITE.erl index 7ccde83745..3f91cc9801 100644 --- a/test/unit_SUITE.erl +++ b/test/unit_SUITE.erl @@ -269,7 +269,7 @@ encrypt_decrypt(_Config) -> %% with a random passphrase. _ = [begin PassPhrase = crypto:strong_rand_bytes(16), - Iterations = rand_compat:uniform(100), + Iterations = rand:uniform(100), Data = crypto:strong_rand_bytes(64), [begin Expected = binary:part(Data, 0, Len), @@ -299,7 +299,7 @@ encrypt_decrypt_term(_Config) -> ], _ = [begin PassPhrase = crypto:strong_rand_bytes(16), - Iterations = rand_compat:uniform(100), + Iterations = rand:uniform(100), Enc = rabbit_pbe:encrypt_term(C, H, Iterations, PassPhrase, Data), Data = rabbit_pbe:decrypt_term(C, H, Iterations, PassPhrase, Enc) end || H <- Hashes, C <- Ciphers, Data <- DataSet], diff --git a/test/unit_inbroker_SUITE.erl b/test/unit_inbroker_SUITE.erl index 98cd77ccbe..7b783facd2 100644 --- a/test/unit_inbroker_SUITE.erl +++ b/test/unit_inbroker_SUITE.erl @@ -116,7 +116,7 @@ groups() -> log_management, %% Check log files. log_management_during_startup, %% Check log files. memory_high_watermark, %% Trigger alarm. - rotate_logs_without_suffix, %% Check log files. + externally_rotated_logs_are_automatically_reopened, %% Check log files. server_status %% Trigger alarm. ]}, {backing_queue_tests, [], [ @@ -1316,7 +1316,7 @@ maybe_switch_queue_mode(VQ) -> random_queue_mode() -> Modes = [lazy, default], - lists:nth(rand_compat:uniform(length(Modes)), Modes). + lists:nth(rand:uniform(length(Modes)), Modes). pub_res({_, VQS}) -> VQS; @@ -1842,69 +1842,74 @@ log_management(Config) -> ?MODULE, log_management1, [Config]). log_management1(_Config) -> - override_group_leader(), + [LogFile] = rabbit:log_locations(), + Suffix = ".0", - MainLog = rabbit:log_location(kernel), - SaslLog = rabbit:log_location(sasl), - Suffix = ".1", - - ok = test_logs_working(MainLog, SaslLog), + ok = test_logs_working([LogFile]), %% prepare basic logs - file:delete([MainLog, Suffix]), - file:delete([SaslLog, Suffix]), - - %% simple logs reopening - ok = control_action(rotate_logs, []), - ok = test_logs_working(MainLog, SaslLog), + file:delete(LogFile ++ Suffix), + ok = test_logs_working([LogFile]), %% simple log rotation - ok = control_action(rotate_logs, [Suffix]), - [true, true] = non_empty_files([[MainLog, Suffix], [SaslLog, Suffix]]), - [true, true] = empty_files([MainLog, SaslLog]), - ok = test_logs_working(MainLog, SaslLog), - - %% reopening logs with log rotation performed first - ok = clean_logs([MainLog, SaslLog], Suffix), ok = control_action(rotate_logs, []), - ok = file:rename(MainLog, [MainLog, Suffix]), - ok = file:rename(SaslLog, [SaslLog, Suffix]), - ok = test_logs_working([MainLog, Suffix], [SaslLog, Suffix]), + %% FIXME: rabbit:rotate_logs/0 is asynchronous due to a limitation + %% in Lager. Therefore, we have no choice but to wait an arbitrary + %% amount of time. + timer:sleep(2000), + [true, true] = non_empty_files([LogFile ++ Suffix, LogFile]), + ok = test_logs_working([LogFile]), + + %% log rotation on empty files + ok = clean_logs([LogFile], Suffix), ok = control_action(rotate_logs, []), - ok = test_logs_working(MainLog, SaslLog), + timer:sleep(2000), + [{error, enoent}, true] = non_empty_files([LogFile ++ Suffix, LogFile]), - %% log rotation on empty files (the main log will have a ctl action logged) - ok = clean_logs([MainLog, SaslLog], Suffix), + %% logs with suffix are not writable + ok = control_action(rotate_logs, []), + timer:sleep(2000), + ok = make_files_non_writable([LogFile ++ Suffix]), ok = control_action(rotate_logs, []), - ok = control_action(rotate_logs, [Suffix]), - [false, true] = empty_files([[MainLog, Suffix], [SaslLog, Suffix]]), + timer:sleep(2000), + ok = test_logs_working([LogFile]), - %% logs with suffix are not writable - ok = control_action(rotate_logs, [Suffix]), - ok = make_files_non_writable([[MainLog, Suffix], [SaslLog, Suffix]]), - ok = control_action(rotate_logs, [Suffix]), - ok = test_logs_working(MainLog, SaslLog), + %% rotate when original log files are not writable + ok = make_files_non_writable([LogFile]), + ok = control_action(rotate_logs, []), + timer:sleep(2000), %% logging directed to tty (first, remove handlers) - ok = delete_log_handlers([rabbit_sasl_report_file_h, - rabbit_error_logger_file_h]), - ok = clean_logs([MainLog, SaslLog], Suffix), - ok = application:set_env(rabbit, sasl_error_logger, tty), - ok = application:set_env(rabbit, error_logger, tty), - ok = control_action(rotate_logs, []), - [{error, enoent}, {error, enoent}] = empty_files([MainLog, SaslLog]), + ok = control_action(stop_app, []), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_handler, tty), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = control_action(start_app, []), + timer:sleep(200), + rabbit_log:info("test info"), + [{error, enoent}] = empty_files([LogFile]), %% rotate logs when logging is turned off - ok = application:set_env(rabbit, sasl_error_logger, false), - ok = application:set_env(rabbit, error_logger, silent), - ok = control_action(rotate_logs, []), - [{error, enoent}, {error, enoent}] = empty_files([MainLog, SaslLog]), + ok = control_action(stop_app, []), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_handler, false), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = control_action(start_app, []), + timer:sleep(200), + rabbit_log:error("test error"), + timer:sleep(200), + [{error, enoent}] = empty_files([LogFile]), %% cleanup - ok = application:set_env(rabbit, sasl_error_logger, {file, SaslLog}), - ok = application:set_env(rabbit, error_logger, {file, MainLog}), - ok = add_log_handlers([{rabbit_error_logger_file_h, MainLog}, - {rabbit_sasl_report_file_h, SaslLog}]), + ok = control_action(stop_app, []), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_handler, LogFile), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = control_action(start_app, []), + ok = test_logs_working([LogFile]), passed. log_management_during_startup(Config) -> @@ -1912,137 +1917,111 @@ log_management_during_startup(Config) -> ?MODULE, log_management_during_startup1, [Config]). log_management_during_startup1(_Config) -> - MainLog = rabbit:log_location(kernel), - SaslLog = rabbit:log_location(sasl), + [LogFile] = rabbit:log_locations(), + Suffix = ".0", %% start application with simple tty logging ok = control_action(stop_app, []), - ok = application:set_env(rabbit, error_logger, tty), - ok = application:set_env(rabbit, sasl_error_logger, tty), - ok = add_log_handlers([{error_logger_tty_h, []}, - {sasl_report_tty_h, []}]), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_handler, tty), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), ok = control_action(start_app, []), - %% start application with tty logging and - %% proper handlers not installed - ok = control_action(stop_app, []), - ok = error_logger:tty(false), - ok = delete_log_handlers([sasl_report_tty_h]), - ok = case catch control_action(start_app, []) of - ok -> exit({got_success_but_expected_failure, - log_rotation_tty_no_handlers_test}); - {badrpc, {'EXIT', {error, - {cannot_log_to_tty, _, not_installed}}}} -> ok - end, - - %% fix sasl logging - ok = application:set_env(rabbit, sasl_error_logger, {file, SaslLog}), - %% start application with logging to non-existing directory - TmpLog = "/tmp/rabbit-tests/test.log", - delete_file(TmpLog), + NonExistent = "/tmp/non-existent/test.log", + delete_file(NonExistent), + delete_file(filename:dirname(NonExistent)), ok = control_action(stop_app, []), - ok = application:set_env(rabbit, error_logger, {file, TmpLog}), - - ok = delete_log_handlers([rabbit_error_logger_file_h]), - ok = add_log_handlers([{error_logger_file_h, MainLog}]), + ok = application:set_env(rabbit, lager_handler, NonExistent), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), ok = control_action(start_app, []), %% start application with logging to directory with no %% write permissions ok = control_action(stop_app, []), - TmpDir = "/tmp/rabbit-tests", - ok = set_permissions(TmpDir, 8#00400), - ok = delete_log_handlers([rabbit_error_logger_file_h]), - ok = add_log_handlers([{error_logger_file_h, MainLog}]), + NoPermission1 = "/var/empty/test.log", + delete_file(NoPermission1), + delete_file(filename:dirname(NoPermission1)), + ok = control_action(stop_app, []), + ok = application:set_env(rabbit, lager_handler, NoPermission1), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), ok = case control_action(start_app, []) of ok -> exit({got_success_but_expected_failure, log_rotation_no_write_permission_dir_test}); - {badrpc, {'EXIT', - {error, {cannot_log_to_file, _, _}}}} -> ok + {badrpc, + {'EXIT', {error, {cannot_log_to_file, _, Reason1}}}} + when Reason1 =:= enoent orelse Reason1 =:= eacces -> ok; + {badrpc, + {'EXIT', + {error, {cannot_log_to_file, _, + {cannot_create_parent_dirs, _, Reason1}}}}} + when Reason1 =:= eperm orelse + Reason1 =:= eacces orelse + Reason1 =:= enoent-> ok end, %% start application with logging to a subdirectory which %% parent directory has no write permissions - ok = control_action(stop_app, []), - TmpTestDir = "/tmp/rabbit-tests/no-permission/test/log", - ok = application:set_env(rabbit, error_logger, {file, TmpTestDir}), - ok = add_log_handlers([{error_logger_file_h, MainLog}]), + NoPermission2 = "/var/empty/non-existent/test.log", + delete_file(NoPermission2), + delete_file(filename:dirname(NoPermission2)), + case control_action(stop_app, []) of + ok -> ok; + {error, lager_not_running} -> ok + end, + ok = application:set_env(rabbit, lager_handler, NoPermission2), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), ok = case control_action(start_app, []) of ok -> exit({got_success_but_expected_failure, log_rotatation_parent_dirs_test}); {badrpc, + {'EXIT', {error, {cannot_log_to_file, _, Reason2}}}} + when Reason2 =:= enoent orelse Reason2 =:= eacces -> ok; + {badrpc, {'EXIT', {error, {cannot_log_to_file, _, - {error, - {cannot_create_parent_dirs, _, eacces}}}}}} -> ok + {cannot_create_parent_dirs, _, Reason2}}}}} + when Reason2 =:= eperm orelse + Reason2 =:= eacces orelse + Reason2 =:= enoent-> ok end, - ok = set_permissions(TmpDir, 8#00700), - ok = set_permissions(TmpLog, 8#00600), - ok = delete_file(TmpLog), - ok = file:del_dir(TmpDir), - %% start application with standard error_logger_file_h - %% handler not installed - ok = control_action(stop_app, []), - ok = application:set_env(rabbit, error_logger, {file, MainLog}), - ok = control_action(start_app, []), - - %% start application with standard sasl handler not installed - %% and rabbit main log handler installed correctly - ok = control_action(stop_app, []), - ok = delete_log_handlers([rabbit_sasl_report_file_h]), + %% cleanup + ok = application:set_env(rabbit, lager_handler, LogFile), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), ok = control_action(start_app, []), passed. -%% "rabbitmqctl rotate_logs" without additional parameters -%% shouldn't truncate files. -rotate_logs_without_suffix(Config) -> +externally_rotated_logs_are_automatically_reopened(Config) -> passed = rabbit_ct_broker_helpers:rpc(Config, 0, - ?MODULE, rotate_logs_without_suffix1, [Config]). - -rotate_logs_without_suffix1(_Config) -> - override_group_leader(), - - MainLog = rabbit:log_location(kernel), - SaslLog = rabbit:log_location(sasl), - Suffix = ".1", - file:delete(MainLog), - file:delete(SaslLog), + ?MODULE, externally_rotated_logs_are_automatically_reopened1, [Config]). - %% Empty log-files should be created - ok = control_action(rotate_logs, []), - [true, true] = empty_files([MainLog, SaslLog]), +externally_rotated_logs_are_automatically_reopened1(_Config) -> + [LogFile] = rabbit:log_locations(), - %% Write something to log files and simulate external log rotation - ok = test_logs_working(MainLog, SaslLog), - ok = file:rename(MainLog, [MainLog, Suffix]), - ok = file:rename(SaslLog, [SaslLog, Suffix]), + %% Make sure log file is opened + ok = test_logs_working([LogFile]), - %% Create non-empty files - TestData = "test-data\n", - file:write_file(MainLog, TestData), - file:write_file(SaslLog, TestData), + %% Move it away - i.e. external log rotation happened + file:rename(LogFile, [LogFile, ".rotation_test"]), - %% Nothing should be truncated - neither moved files which are still - %% opened by server, nor new log files that should be just reopened. - ok = control_action(rotate_logs, []), - [true, true, true, true] = - non_empty_files([MainLog, SaslLog, [MainLog, Suffix], - [SaslLog, Suffix]]), - - %% And log files should be re-opened - new log records should go to - %% new files. - ok = test_logs_working(MainLog, SaslLog), - true = (rabbit_file:file_size(MainLog) > length(TestData)), - true = (rabbit_file:file_size(SaslLog) > length(TestData)), + %% New files should be created - test_logs_working/1 will check that + %% LogFile is not empty after doing some logging. And it's exactly + %% what we need to check here. + ok = test_logs_working([LogFile]), passed. -override_group_leader() -> - %% Override group leader, otherwise SASL fake events are ignored by - %% the error_logger local to RabbitMQ. - {group_leader, Leader} = erlang:process_info(whereis(rabbit), group_leader), - erlang:group_leader(Leader, self()). +empty_or_nonexist_files(Files) -> + [case file:read_file_info(File) of + {ok, FInfo} -> FInfo#file_info.size == 0; + {error, enoent} -> true; + Error -> Error + end || File <- Files]. empty_files(Files) -> [case file:read_file_info(File) of @@ -2056,12 +2035,11 @@ non_empty_files(Files) -> _ -> not(EmptyFile) end || EmptyFile <- empty_files(Files)]. -test_logs_working(MainLogFile, SaslLogFile) -> - ok = rabbit_log:error("Log a test message~n"), - ok = error_logger:error_report(crash_report, [fake_crash_report, ?MODULE]), +test_logs_working(LogFiles) -> + ok = rabbit_log:error("Log a test message"), %% give the error loggers some time to catch up - timer:sleep(100), - [true, true] = non_empty_files([MainLogFile, SaslLogFile]), + timer:sleep(200), + lists:all(fun(LogFile) -> [true] =:= non_empty_files([LogFile]) end, LogFiles), ok. set_permissions(Path, Mode) -> |