diff options
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | README.md | 8 | ||||
| -rw-r--r-- | docs/rabbitmq.config.example | 21 | ||||
| -rw-r--r-- | docs/rabbitmqctl.1.xml | 73 | ||||
| -rw-r--r-- | erlang.mk | 727 | ||||
| -rw-r--r-- | include/rabbit_cli.hrl | 13 | ||||
| -rw-r--r-- | rabbitmq-components.mk | 4 | ||||
| -rw-r--r-- | src/gm.erl | 19 | ||||
| -rw-r--r-- | src/pg2_fixed.erl | 399 | ||||
| -rw-r--r-- | src/rabbit.app.src | 10 | ||||
| -rw-r--r-- | src/rabbit.erl | 106 | ||||
| -rw-r--r-- | src/rabbit_amqqueue_process.erl | 8 | ||||
| -rw-r--r-- | src/rabbit_control_main.erl | 16 | ||||
| -rw-r--r-- | src/rabbit_control_pbe.erl | 79 | ||||
| -rw-r--r-- | src/rabbit_mirror_queue_mode_nodes.erl | 36 | ||||
| -rw-r--r-- | src/rabbit_mirror_queue_slave.erl | 7 | ||||
| -rw-r--r-- | src/rabbit_variable_queue.erl | 149 | ||||
| -rw-r--r-- | test/dynamic_ha_SUITE.erl | 55 | ||||
| -rw-r--r-- | test/health_check_SUITE.erl | 48 | ||||
| -rw-r--r-- | test/partitions_SUITE.erl | 16 | ||||
| -rw-r--r-- | test/unit_SUITE.erl | 244 | ||||
| -rw-r--r-- | test/unit_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app | 46 | ||||
| -rw-r--r-- | test/unit_SUITE_data/rabbit_shovel_test.passphrase | 1 |
23 files changed, 1267 insertions, 820 deletions
@@ -2,7 +2,7 @@ PROJECT = rabbit VERSION ?= $(call get_app_version,src/$(PROJECT).app.src) DEPS = ranch rabbit_common -TEST_DEPS = rabbitmq_ct_helpers amqp_client meck proper +TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper define usage_xml_to_erl $(subst __,_,$(patsubst $(DOCS_DIR)/rabbitmq%.1.xml, src/rabbit_%_usage.erl, $(subst -,_,$(1)))) @@ -17,8 +17,9 @@ * [RabbitMQ tutorials](http://www.rabbitmq.com/getstarted.html) * [Documentation guides](http://www.rabbitmq.com/documentation.html) + * [Documentation Source Code](https://github.com/rabbitmq/rabbitmq-website/) * [Client libraries and tools](http://www.rabbitmq.com/devtools.html) - + * [Tutorials Source Code](https://github.com/rabbitmq/rabbitmq-tutorials/) ## Getting Help @@ -37,9 +38,10 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) and our [development process overview]( RabbitMQ server is [licensed under the MPL](LICENSE-MPL-RabbitMQ). -## Building From Source +## Building From Source and Packaging -See [building RabbitMQ server from source](http://www.rabbitmq.com/build-server.html). + * [Building RabbitMQ Server From Source](http://www.rabbitmq.com/build-server.html) + * [Building RabbitMQ Server Packages](http://www.rabbitmq.com/build-server.html) ## Copyright diff --git a/docs/rabbitmq.config.example b/docs/rabbitmq.config.example index 4d376d953a..f425726721 100644 --- a/docs/rabbitmq.config.example +++ b/docs/rabbitmq.config.example @@ -132,6 +132,27 @@ %% %% {password_hashing_module, rabbit_password_hashing_sha256}, + %% Configuration entry encryption. + %% See http://www.rabbitmq.com/configure.html#configuration-encryption + %% + %% To specify the passphrase in the configuration file: + %% + %% {config_entry_decoder, [{passphrase, <<"mypassphrase">>}]} + %% + %% To specify the passphrase in an external file: + %% + %% {config_entry_decoder, [{passphrase, {file, "/path/to/passphrase/file"}}]} + %% + %% To make the broker request the passphrase when it starts: + %% + %% {config_entry_decoder, [{passphrase, prompt}]} + %% + %% To change encryption settings: + %% + %% {config_entry_decoder, [{cipher, aes_cbc256}, + %% {hash, sha512}, + %% {iterations, 1000}]} + %% %% Default User / VHost %% ==================== diff --git a/docs/rabbitmqctl.1.xml b/docs/rabbitmqctl.1.xml index d9a9991ea3..217d2d93ca 100644 --- a/docs/rabbitmqctl.1.xml +++ b/docs/rabbitmqctl.1.xml @@ -2083,6 +2083,79 @@ </variablelist> </listitem> </varlistentry> + <varlistentry> + <!-- one-line formatting matters for rabbit_ctl_usage.erl code generation --> + <term><cmdsynopsis><command>encode</command> <arg choice="opt">--decode</arg> <arg choice="opt"><replaceable>value</replaceable></arg> <arg choice="opt"><replaceable>passphrase</replaceable></arg> <arg choice="opt">--list-ciphers</arg> <arg choice="opt">--list-hashes</arg> <arg choice="opt">--cipher <replaceable>cipher</replaceable></arg> <arg choice="opt">--hash <replaceable>hash</replaceable></arg> <arg choice="opt">--iterations <replaceable>iterations</replaceable></arg></cmdsynopsis> + </term> + <listitem> + <variablelist> + <varlistentry> + <term><cmdsynopsis><arg choice="opt">--decode</arg></cmdsynopsis></term> + <listitem> + <para> + Flag to decrypt the input value. + </para> + <para role="example-prefix">For example:</para> + <screen role="example">rabbitmqctl encode --decode '{encrypted,'<<"...">>}' mypassphrase</screen> + </listitem> + </varlistentry> + <varlistentry> + <term> + <cmdsynopsis> + <arg choice="opt"><replaceable>value</replaceable></arg> + <arg choice="opt"><replaceable>passphrase</replaceable></arg> + </cmdsynopsis> + </term> + <listitem> + <para> + Value to encrypt/decrypt and passphrase. + </para> + <para role="example-prefix">For example:</para> + <screen role="example">rabbitmqctl encode '<<"guest">>' mypassphrase</screen> + <screen role="example">rabbitmqctl encode --decode '{encrypted,'<<"...">>}' mypassphrase</screen> + </listitem> + </varlistentry> + <varlistentry> + <term><cmdsynopsis><arg choice="opt">--list-ciphers</arg></cmdsynopsis></term> + <listitem> + <para> + Flag to list the supported ciphers. + </para> + <para role="example-prefix">For example:</para> + <screen role="example">rabbitmqctl encode --list-ciphers</screen> + </listitem> + </varlistentry> + <varlistentry> + <term><cmdsynopsis><arg choice="opt">--list-hashes</arg></cmdsynopsis></term> + <listitem> + <para> + Flag to list the supported hash algorithms. + </para> + <para role="example-prefix">For example:</para> + <screen role="example">rabbitmqctl encode --list-hashes</screen> + </listitem> + </varlistentry> + <varlistentry> + <term> + <cmdsynopsis> + <arg choice="opt">--cipher <replaceable>cipher</replaceable></arg> + <arg choice="opt">--hash <replaceable>hash</replaceable></arg> + <arg choice="opt">--iterations <replaceable>iterations</replaceable></arg> + </cmdsynopsis> + </term> + <listitem> + <para> + Options to specify the encryption settings. They can be used independently. + </para> + <para role="example-prefix">For example:</para> + <screen role="example"> +rabbitmqctl encode --cipher blowfish_cfb64 --hash sha256 --iterations 10000 \ + '<<"guest">>' mypassphrase</screen> + </listitem> + </varlistentry> + </variablelist> + </listitem> + </varlistentry> </variablelist> </refsect2> </refsect1> @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -12,11 +12,21 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -.PHONY: all app apps deps search rel docs install-docs check tests clean distclean help erlang-mk +.PHONY: all app apps deps search rel relup docs install-docs check tests clean distclean help erlang-mk ERLANG_MK_FILENAME := $(realpath $(lastword $(MAKEFILE_LIST))) -ERLANG_MK_VERSION = 2.0.0-pre.2-144-g647ffd1 +ERLANG_MK_VERSION = 2.0.0-pre.2-207-g9e9b7d2 + +# Make 3.81 and 3.82 are deprecated. + +ifeq ($(MAKE_VERSION),3.81) +$(warning Please upgrade to GNU Make 4 or later: https://erlang.mk/guide/installation.html) +endif + +ifeq ($(MAKE_VERSION),3.82) +$(warning Please upgrade to GNU Make 4 or later: https://erlang.mk/guide/installation.html) +endif # Core configuration. @@ -25,6 +35,7 @@ PROJECT := $(strip $(PROJECT)) PROJECT_VERSION ?= rolling PROJECT_MOD ?= $(PROJECT)_app +PROJECT_ENV ?= [] # Verbosity. @@ -85,6 +96,8 @@ all:: deps app rel rel:: $(verbose) : +relup:: deps app + check:: tests clean:: clean-crashdump @@ -102,7 +115,7 @@ distclean-tmp: help:: $(verbose) printf "%s\n" \ "erlang.mk (version $(ERLANG_MK_VERSION)) is distributed under the terms of the ISC License." \ - "Copyright (c) 2013-2015 Loïc Hoguin <essen@ninenines.eu>" \ + "Copyright (c) 2013-2016 Loïc Hoguin <essen@ninenines.eu>" \ "" \ "Usage: [V=1] $(MAKE) [target]..." \ "" \ @@ -150,30 +163,7 @@ else core_native_path = $1 endif -ifeq ($(shell which wget 2>/dev/null | wc -l), 1) -define core_http_get - wget --no-check-certificate -O $(1) $(2)|| rm $(1) -endef -else -define core_http_get.erl - ssl:start(), - inets:start(), - case httpc:request(get, {"$(2)", []}, [{autoredirect, true}], []) of - {ok, {{_, 200, _}, _, Body}} -> - case file:write_file("$(1)", Body) of - ok -> ok; - {error, R1} -> halt(R1) - end; - {error, R2} -> - halt(R2) - end, - halt(0). -endef - -define core_http_get - $(call erlang,$(call core_http_get.erl,$(call core_native_path,$1),$2)) -endef -endif +core_http_get = curl -Lf$(if $(filter-out 0,$(V)),,s)o $(call core_native_path,$1) $2 core_eq = $(and $(findstring $(1),$(2)),$(findstring $(2),$(1))) @@ -286,6 +276,14 @@ pkg_apns_fetch = git pkg_apns_repo = https://github.com/inaka/apns4erl pkg_apns_commit = master +PACKAGES += asciideck +pkg_asciideck_name = asciideck +pkg_asciideck_description = Asciidoc for Erlang. +pkg_asciideck_homepage = https://ninenines.eu +pkg_asciideck_fetch = git +pkg_asciideck_repo = https://github.com/ninenines/asciideck +pkg_asciideck_commit = master + PACKAGES += azdht pkg_azdht_name = azdht pkg_azdht_description = Azureus Distributed Hash Table (DHT) in Erlang @@ -406,14 +404,6 @@ pkg_bootstrap_fetch = git pkg_bootstrap_repo = https://github.com/schlagert/bootstrap pkg_bootstrap_commit = master -PACKAGES += boss_db -pkg_boss_db_name = boss_db -pkg_boss_db_description = BossDB: a sharded, caching, pooling, evented ORM for Erlang -pkg_boss_db_homepage = https://github.com/ErlyORM/boss_db -pkg_boss_db_fetch = git -pkg_boss_db_repo = https://github.com/ErlyORM/boss_db -pkg_boss_db_commit = master - PACKAGES += boss pkg_boss_name = boss pkg_boss_description = Erlang web MVC, now featuring Comet @@ -422,6 +412,14 @@ pkg_boss_fetch = git pkg_boss_repo = https://github.com/ChicagoBoss/ChicagoBoss pkg_boss_commit = master +PACKAGES += boss_db +pkg_boss_db_name = boss_db +pkg_boss_db_description = BossDB: a sharded, caching, pooling, evented ORM for Erlang +pkg_boss_db_homepage = https://github.com/ErlyORM/boss_db +pkg_boss_db_fetch = git +pkg_boss_db_repo = https://github.com/ErlyORM/boss_db +pkg_boss_db_commit = master + PACKAGES += brod pkg_brod_name = brod pkg_brod_description = Kafka client in Erlang @@ -566,13 +564,13 @@ pkg_cloudi_service_api_requests_fetch = git pkg_cloudi_service_api_requests_repo = https://github.com/CloudI/cloudi_service_api_requests pkg_cloudi_service_api_requests_commit = master -PACKAGES += cloudi_service_db_cassandra_cql -pkg_cloudi_service_db_cassandra_cql_name = cloudi_service_db_cassandra_cql -pkg_cloudi_service_db_cassandra_cql_description = Cassandra CQL CloudI Service -pkg_cloudi_service_db_cassandra_cql_homepage = http://cloudi.org/ -pkg_cloudi_service_db_cassandra_cql_fetch = git -pkg_cloudi_service_db_cassandra_cql_repo = https://github.com/CloudI/cloudi_service_db_cassandra_cql -pkg_cloudi_service_db_cassandra_cql_commit = master +PACKAGES += cloudi_service_db +pkg_cloudi_service_db_name = cloudi_service_db +pkg_cloudi_service_db_description = CloudI Database (in-memory/testing/generic) +pkg_cloudi_service_db_homepage = http://cloudi.org/ +pkg_cloudi_service_db_fetch = git +pkg_cloudi_service_db_repo = https://github.com/CloudI/cloudi_service_db +pkg_cloudi_service_db_commit = master PACKAGES += cloudi_service_db_cassandra pkg_cloudi_service_db_cassandra_name = cloudi_service_db_cassandra @@ -582,6 +580,14 @@ pkg_cloudi_service_db_cassandra_fetch = git pkg_cloudi_service_db_cassandra_repo = https://github.com/CloudI/cloudi_service_db_cassandra pkg_cloudi_service_db_cassandra_commit = master +PACKAGES += cloudi_service_db_cassandra_cql +pkg_cloudi_service_db_cassandra_cql_name = cloudi_service_db_cassandra_cql +pkg_cloudi_service_db_cassandra_cql_description = Cassandra CQL CloudI Service +pkg_cloudi_service_db_cassandra_cql_homepage = http://cloudi.org/ +pkg_cloudi_service_db_cassandra_cql_fetch = git +pkg_cloudi_service_db_cassandra_cql_repo = https://github.com/CloudI/cloudi_service_db_cassandra_cql +pkg_cloudi_service_db_cassandra_cql_commit = master + PACKAGES += cloudi_service_db_couchdb pkg_cloudi_service_db_couchdb_name = cloudi_service_db_couchdb pkg_cloudi_service_db_couchdb_description = CouchDB CloudI Service @@ -638,14 +644,6 @@ pkg_cloudi_service_db_tokyotyrant_fetch = git pkg_cloudi_service_db_tokyotyrant_repo = https://github.com/CloudI/cloudi_service_db_tokyotyrant pkg_cloudi_service_db_tokyotyrant_commit = master -PACKAGES += cloudi_service_db -pkg_cloudi_service_db_name = cloudi_service_db -pkg_cloudi_service_db_description = CloudI Database (in-memory/testing/generic) -pkg_cloudi_service_db_homepage = http://cloudi.org/ -pkg_cloudi_service_db_fetch = git -pkg_cloudi_service_db_repo = https://github.com/CloudI/cloudi_service_db -pkg_cloudi_service_db_commit = master - PACKAGES += cloudi_service_filesystem pkg_cloudi_service_filesystem_name = cloudi_service_filesystem pkg_cloudi_service_filesystem_description = Filesystem CloudI Service @@ -1038,14 +1036,6 @@ pkg_edown_fetch = git pkg_edown_repo = https://github.com/uwiger/edown pkg_edown_commit = master -PACKAGES += eep_app -pkg_eep_app_name = eep_app -pkg_eep_app_description = Embedded Event Processing -pkg_eep_app_homepage = https://github.com/darach/eep-erl -pkg_eep_app_fetch = git -pkg_eep_app_repo = https://github.com/darach/eep-erl -pkg_eep_app_commit = master - PACKAGES += eep pkg_eep_name = eep pkg_eep_description = Erlang Easy Profiling (eep) application provides a way to analyze application performance and call hierarchy @@ -1054,6 +1044,14 @@ pkg_eep_fetch = git pkg_eep_repo = https://github.com/virtan/eep pkg_eep_commit = master +PACKAGES += eep_app +pkg_eep_app_name = eep_app +pkg_eep_app_description = Embedded Event Processing +pkg_eep_app_homepage = https://github.com/darach/eep-erl +pkg_eep_app_fetch = git +pkg_eep_app_repo = https://github.com/darach/eep-erl +pkg_eep_app_commit = master + PACKAGES += efene pkg_efene_name = efene pkg_efene_description = Alternative syntax for the Erlang Programming Language focusing on simplicity, ease of use and programmer UX @@ -1238,14 +1236,6 @@ pkg_eqm_fetch = git pkg_eqm_repo = https://github.com/loucash/eqm pkg_eqm_commit = master -PACKAGES += eredis_pool -pkg_eredis_pool_name = eredis_pool -pkg_eredis_pool_description = eredis_pool is Pool of Redis clients, using eredis and poolboy. -pkg_eredis_pool_homepage = https://github.com/hiroeorz/eredis_pool -pkg_eredis_pool_fetch = git -pkg_eredis_pool_repo = https://github.com/hiroeorz/eredis_pool -pkg_eredis_pool_commit = master - PACKAGES += eredis pkg_eredis_name = eredis pkg_eredis_description = Erlang Redis client @@ -1254,6 +1244,14 @@ pkg_eredis_fetch = git pkg_eredis_repo = https://github.com/wooga/eredis pkg_eredis_commit = master +PACKAGES += eredis_pool +pkg_eredis_pool_name = eredis_pool +pkg_eredis_pool_description = eredis_pool is Pool of Redis clients, using eredis and poolboy. +pkg_eredis_pool_homepage = https://github.com/hiroeorz/eredis_pool +pkg_eredis_pool_fetch = git +pkg_eredis_pool_repo = https://github.com/hiroeorz/eredis_pool +pkg_eredis_pool_commit = master + PACKAGES += erl_streams pkg_erl_streams_name = erl_streams pkg_erl_streams_description = Streams in Erlang @@ -1542,14 +1540,6 @@ pkg_etap_fetch = git pkg_etap_repo = https://github.com/ngerakines/etap pkg_etap_commit = master -PACKAGES += etest_http -pkg_etest_http_name = etest_http -pkg_etest_http_description = etest Assertions around HTTP (client-side) -pkg_etest_http_homepage = https://github.com/wooga/etest_http -pkg_etest_http_fetch = git -pkg_etest_http_repo = https://github.com/wooga/etest_http -pkg_etest_http_commit = master - PACKAGES += etest pkg_etest_name = etest pkg_etest_description = A lightweight, convention over configuration test framework for Erlang @@ -1558,6 +1548,14 @@ pkg_etest_fetch = git pkg_etest_repo = https://github.com/wooga/etest pkg_etest_commit = master +PACKAGES += etest_http +pkg_etest_http_name = etest_http +pkg_etest_http_description = etest Assertions around HTTP (client-side) +pkg_etest_http_homepage = https://github.com/wooga/etest_http +pkg_etest_http_fetch = git +pkg_etest_http_repo = https://github.com/wooga/etest_http +pkg_etest_http_commit = master + PACKAGES += etoml pkg_etoml_name = etoml pkg_etoml_description = TOML language erlang parser @@ -1566,14 +1564,6 @@ pkg_etoml_fetch = git pkg_etoml_repo = https://github.com/kalta/etoml pkg_etoml_commit = master -PACKAGES += eunit_formatters -pkg_eunit_formatters_name = eunit_formatters -pkg_eunit_formatters_description = Because eunit's output sucks. Let's make it better. -pkg_eunit_formatters_homepage = https://github.com/seancribbs/eunit_formatters -pkg_eunit_formatters_fetch = git -pkg_eunit_formatters_repo = https://github.com/seancribbs/eunit_formatters -pkg_eunit_formatters_commit = master - PACKAGES += eunit pkg_eunit_name = eunit pkg_eunit_description = The EUnit lightweight unit testing framework for Erlang - this is the canonical development repository. @@ -1582,6 +1572,14 @@ pkg_eunit_fetch = git pkg_eunit_repo = https://github.com/richcarl/eunit pkg_eunit_commit = master +PACKAGES += eunit_formatters +pkg_eunit_formatters_name = eunit_formatters +pkg_eunit_formatters_description = Because eunit's output sucks. Let's make it better. +pkg_eunit_formatters_homepage = https://github.com/seancribbs/eunit_formatters +pkg_eunit_formatters_fetch = git +pkg_eunit_formatters_repo = https://github.com/seancribbs/eunit_formatters +pkg_eunit_formatters_commit = master + PACKAGES += euthanasia pkg_euthanasia_name = euthanasia pkg_euthanasia_description = Merciful killer for your Erlang processes @@ -1599,7 +1597,7 @@ pkg_evum_repo = https://github.com/msantos/evum pkg_evum_commit = master PACKAGES += exec -pkg_exec_name = exec +pkg_exec_name = erlexec pkg_exec_description = Execute and control OS processes from Erlang/OTP. pkg_exec_homepage = http://saleyn.github.com/erlexec pkg_exec_fetch = git @@ -1718,14 +1716,6 @@ pkg_fn_fetch = git pkg_fn_repo = https://github.com/reiddraper/fn pkg_fn_commit = master -PACKAGES += folsom_cowboy -pkg_folsom_cowboy_name = folsom_cowboy -pkg_folsom_cowboy_description = A Cowboy based Folsom HTTP Wrapper. -pkg_folsom_cowboy_homepage = https://github.com/boundary/folsom_cowboy -pkg_folsom_cowboy_fetch = git -pkg_folsom_cowboy_repo = https://github.com/boundary/folsom_cowboy -pkg_folsom_cowboy_commit = master - PACKAGES += folsom pkg_folsom_name = folsom pkg_folsom_description = Expose Erlang Events and Metrics @@ -1734,6 +1724,14 @@ pkg_folsom_fetch = git pkg_folsom_repo = https://github.com/boundary/folsom pkg_folsom_commit = master +PACKAGES += folsom_cowboy +pkg_folsom_cowboy_name = folsom_cowboy +pkg_folsom_cowboy_description = A Cowboy based Folsom HTTP Wrapper. +pkg_folsom_cowboy_homepage = https://github.com/boundary/folsom_cowboy +pkg_folsom_cowboy_fetch = git +pkg_folsom_cowboy_repo = https://github.com/boundary/folsom_cowboy +pkg_folsom_cowboy_commit = master + PACKAGES += folsomite pkg_folsomite_name = folsomite pkg_folsomite_description = blow up your graphite / riemann server with folsom metrics @@ -2094,14 +2092,6 @@ pkg_jesse_fetch = git pkg_jesse_repo = https://github.com/for-GET/jesse pkg_jesse_commit = master -PACKAGES += jiffy_v -pkg_jiffy_v_name = jiffy_v -pkg_jiffy_v_description = JSON validation utility -pkg_jiffy_v_homepage = https://github.com/shizzard/jiffy-v -pkg_jiffy_v_fetch = git -pkg_jiffy_v_repo = https://github.com/shizzard/jiffy-v -pkg_jiffy_v_commit = master - PACKAGES += jiffy pkg_jiffy_name = jiffy pkg_jiffy_description = JSON NIFs for Erlang. @@ -2110,6 +2100,14 @@ pkg_jiffy_fetch = git pkg_jiffy_repo = https://github.com/davisp/jiffy pkg_jiffy_commit = master +PACKAGES += jiffy_v +pkg_jiffy_v_name = jiffy_v +pkg_jiffy_v_description = JSON validation utility +pkg_jiffy_v_homepage = https://github.com/shizzard/jiffy-v +pkg_jiffy_v_fetch = git +pkg_jiffy_v_repo = https://github.com/shizzard/jiffy-v +pkg_jiffy_v_commit = master + PACKAGES += jobs pkg_jobs_name = jobs pkg_jobs_description = a Job scheduler for load regulation @@ -2126,14 +2124,6 @@ pkg_joxa_fetch = git pkg_joxa_repo = https://github.com/joxa/joxa pkg_joxa_commit = master -PACKAGES += json_rec -pkg_json_rec_name = json_rec -pkg_json_rec_description = JSON to erlang record -pkg_json_rec_homepage = https://github.com/justinkirby/json_rec -pkg_json_rec_fetch = git -pkg_json_rec_repo = https://github.com/justinkirby/json_rec -pkg_json_rec_commit = master - PACKAGES += json pkg_json_name = json pkg_json_description = a high level json library for erlang (17.0+) @@ -2142,6 +2132,14 @@ pkg_json_fetch = git pkg_json_repo = https://github.com/talentdeficit/json pkg_json_commit = master +PACKAGES += json_rec +pkg_json_rec_name = json_rec +pkg_json_rec_description = JSON to erlang record +pkg_json_rec_homepage = https://github.com/justinkirby/json_rec +pkg_json_rec_fetch = git +pkg_json_rec_repo = https://github.com/justinkirby/json_rec +pkg_json_rec_commit = master + PACKAGES += jsone pkg_jsone_name = jsone pkg_jsone_description = An Erlang library for encoding, decoding JSON data. @@ -2182,14 +2180,6 @@ pkg_jsx_fetch = git pkg_jsx_repo = https://github.com/talentdeficit/jsx pkg_jsx_commit = master -PACKAGES += kafka_protocol -pkg_kafka_protocol_name = kafka_protocol -pkg_kafka_protocol_description = Kafka protocol Erlang library -pkg_kafka_protocol_homepage = https://github.com/klarna/kafka_protocol -pkg_kafka_protocol_fetch = git -pkg_kafka_protocol_repo = https://github.com/klarna/kafka_protocol.git -pkg_kafka_protocol_commit = master - PACKAGES += kafka pkg_kafka_name = kafka pkg_kafka_description = Kafka consumer and producer in Erlang @@ -2198,6 +2188,14 @@ pkg_kafka_fetch = git pkg_kafka_repo = https://github.com/wooga/kafka-erlang pkg_kafka_commit = master +PACKAGES += kafka_protocol +pkg_kafka_protocol_name = kafka_protocol +pkg_kafka_protocol_description = Kafka protocol Erlang library +pkg_kafka_protocol_homepage = https://github.com/klarna/kafka_protocol +pkg_kafka_protocol_fetch = git +pkg_kafka_protocol_repo = https://github.com/klarna/kafka_protocol.git +pkg_kafka_protocol_commit = master + PACKAGES += kai pkg_kai_name = kai pkg_kai_description = DHT storage by Takeshi Inoue @@ -2294,6 +2292,14 @@ pkg_kvs_fetch = git pkg_kvs_repo = https://github.com/synrc/kvs pkg_kvs_commit = master +PACKAGES += lager +pkg_lager_name = lager +pkg_lager_description = A logging framework for Erlang/OTP. +pkg_lager_homepage = https://github.com/basho/lager +pkg_lager_fetch = git +pkg_lager_repo = https://github.com/basho/lager +pkg_lager_commit = master + PACKAGES += lager_amqp_backend pkg_lager_amqp_backend_name = lager_amqp_backend pkg_lager_amqp_backend_description = AMQP RabbitMQ Lager backend @@ -2310,14 +2316,6 @@ pkg_lager_syslog_fetch = git pkg_lager_syslog_repo = https://github.com/basho/lager_syslog pkg_lager_syslog_commit = master -PACKAGES += lager -pkg_lager_name = lager -pkg_lager_description = A logging framework for Erlang/OTP. -pkg_lager_homepage = https://github.com/basho/lager -pkg_lager_fetch = git -pkg_lager_repo = https://github.com/basho/lager -pkg_lager_commit = master - PACKAGES += lambdapad pkg_lambdapad_name = lambdapad pkg_lambdapad_description = Static site generator using Erlang. Yes, Erlang. @@ -2574,14 +2572,6 @@ pkg_mixer_fetch = git pkg_mixer_repo = https://github.com/chef/mixer pkg_mixer_commit = master -PACKAGES += mochiweb_xpath -pkg_mochiweb_xpath_name = mochiweb_xpath -pkg_mochiweb_xpath_description = XPath support for mochiweb's html parser -pkg_mochiweb_xpath_homepage = https://github.com/retnuh/mochiweb_xpath -pkg_mochiweb_xpath_fetch = git -pkg_mochiweb_xpath_repo = https://github.com/retnuh/mochiweb_xpath -pkg_mochiweb_xpath_commit = master - PACKAGES += mochiweb pkg_mochiweb_name = mochiweb pkg_mochiweb_description = MochiWeb is an Erlang library for building lightweight HTTP servers. @@ -2590,6 +2580,14 @@ pkg_mochiweb_fetch = git pkg_mochiweb_repo = https://github.com/mochi/mochiweb pkg_mochiweb_commit = master +PACKAGES += mochiweb_xpath +pkg_mochiweb_xpath_name = mochiweb_xpath +pkg_mochiweb_xpath_description = XPath support for mochiweb's html parser +pkg_mochiweb_xpath_homepage = https://github.com/retnuh/mochiweb_xpath +pkg_mochiweb_xpath_fetch = git +pkg_mochiweb_xpath_repo = https://github.com/retnuh/mochiweb_xpath +pkg_mochiweb_xpath_commit = master + PACKAGES += mockgyver pkg_mockgyver_name = mockgyver pkg_mockgyver_description = A mocking library for Erlang @@ -3062,14 +3060,6 @@ pkg_quickrand_fetch = git pkg_quickrand_repo = https://github.com/okeuday/quickrand pkg_quickrand_commit = master -PACKAGES += rabbit_exchange_type_riak -pkg_rabbit_exchange_type_riak_name = rabbit_exchange_type_riak -pkg_rabbit_exchange_type_riak_description = Custom RabbitMQ exchange type for sticking messages in Riak -pkg_rabbit_exchange_type_riak_homepage = https://github.com/jbrisbin/riak-exchange -pkg_rabbit_exchange_type_riak_fetch = git -pkg_rabbit_exchange_type_riak_repo = https://github.com/jbrisbin/riak-exchange -pkg_rabbit_exchange_type_riak_commit = master - PACKAGES += rabbit pkg_rabbit_name = rabbit pkg_rabbit_description = RabbitMQ Server @@ -3078,6 +3068,14 @@ pkg_rabbit_fetch = git pkg_rabbit_repo = https://github.com/rabbitmq/rabbitmq-server.git pkg_rabbit_commit = master +PACKAGES += rabbit_exchange_type_riak +pkg_rabbit_exchange_type_riak_name = rabbit_exchange_type_riak +pkg_rabbit_exchange_type_riak_description = Custom RabbitMQ exchange type for sticking messages in Riak +pkg_rabbit_exchange_type_riak_homepage = https://github.com/jbrisbin/riak-exchange +pkg_rabbit_exchange_type_riak_fetch = git +pkg_rabbit_exchange_type_riak_repo = https://github.com/jbrisbin/riak-exchange +pkg_rabbit_exchange_type_riak_commit = master + PACKAGES += rack pkg_rack_name = rack pkg_rack_description = Rack handler for erlang @@ -3494,6 +3492,14 @@ pkg_smother_fetch = git pkg_smother_repo = https://github.com/ramsay-t/Smother pkg_smother_commit = master +PACKAGES += snappyer +pkg_snappyer_name = snappyer +pkg_snappyer_description = Snappy as nif for Erlang +pkg_snappyer_homepage = https://github.com/zmstone/snappyer +pkg_snappyer_fetch = git +pkg_snappyer_repo = https://github.com/zmstone/snappyer.git +pkg_snappyer_commit = master + PACKAGES += social pkg_social_name = social pkg_social_description = Cowboy handler for social login via OAuth2 providers @@ -3542,14 +3548,6 @@ pkg_stable_fetch = git pkg_stable_repo = https://github.com/dvv/stable pkg_stable_commit = master -PACKAGES += statebox_riak -pkg_statebox_riak_name = statebox_riak -pkg_statebox_riak_description = Convenience library that makes it easier to use statebox with riak, extracted from best practices in our production code at Mochi Media. -pkg_statebox_riak_homepage = https://github.com/mochi/statebox_riak -pkg_statebox_riak_fetch = git -pkg_statebox_riak_repo = https://github.com/mochi/statebox_riak -pkg_statebox_riak_commit = master - PACKAGES += statebox pkg_statebox_name = statebox pkg_statebox_description = Erlang state monad with merge/conflict-resolution capabilities. Useful for Riak. @@ -3558,6 +3556,14 @@ pkg_statebox_fetch = git pkg_statebox_repo = https://github.com/mochi/statebox pkg_statebox_commit = master +PACKAGES += statebox_riak +pkg_statebox_riak_name = statebox_riak +pkg_statebox_riak_description = Convenience library that makes it easier to use statebox with riak, extracted from best practices in our production code at Mochi Media. +pkg_statebox_riak_homepage = https://github.com/mochi/statebox_riak +pkg_statebox_riak_fetch = git +pkg_statebox_riak_repo = https://github.com/mochi/statebox_riak +pkg_statebox_riak_commit = master + PACKAGES += statman pkg_statman_name = statman pkg_statman_description = Efficiently collect massive volumes of metrics inside the Erlang VM @@ -4062,7 +4068,7 @@ pkg_zucchini_fetch = git pkg_zucchini_repo = https://github.com/devinus/zucchini pkg_zucchini_commit = master -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: search @@ -4089,7 +4095,7 @@ else $(foreach p,$(PACKAGES),$(call pkg_print,$(p))) endif -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: distclean-deps @@ -4166,6 +4172,9 @@ endif ifneq ($(SKIP_DEPS),) deps:: else +ifeq ($(ALL_DEPS_DIRS),) +deps:: apps +else deps:: $(ALL_DEPS_DIRS) apps ifeq ($(IS_APP)$(IS_DEP),) $(verbose) rm -f $(ERLANG_MK_TMP)/deps.log @@ -4185,6 +4194,7 @@ endif fi \ done endif +endif # Deps related targets. @@ -4254,7 +4264,7 @@ define dep_autopatch_fetch_rebar if [ ! -d $(ERLANG_MK_TMP)/rebar ]; then \ git clone -q -n -- https://github.com/rebar/rebar $(ERLANG_MK_TMP)/rebar; \ cd $(ERLANG_MK_TMP)/rebar; \ - git checkout -q 791db716b5a3a7671e0b351f95ddf24b848ee173; \ + git checkout -q 576e12171ab8d69b048b827b92aa65d067deea01; \ $(MAKE); \ cd -; \ fi @@ -4271,6 +4281,7 @@ endef define dep_autopatch_rebar.erl application:load(rebar), application:set_env(rebar, log_level, debug), + rmemo:start(), Conf1 = case file:consult("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config)") of {ok, Conf0} -> Conf0; _ -> [] @@ -4424,9 +4435,9 @@ define dep_autopatch_rebar.erl [] -> ok; _ -> Write("\npre-app::\n\t$$\(MAKE) -f c_src/Makefile.erlang.mk\n"), - PortSpecWrite(io_lib:format("ERL_CFLAGS = -finline-functions -Wall -fPIC -I \\"~s/erts-~s/include\\" -I \\"~s\\"\n", + PortSpecWrite(io_lib:format("ERL_CFLAGS ?= -finline-functions -Wall -fPIC -I \\"~s/erts-~s/include\\" -I \\"~s\\"\n", [code:root_dir(), erlang:system_info(version), code:lib_dir(erl_interface, include)])), - PortSpecWrite(io_lib:format("ERL_LDFLAGS = -L \\"~s\\" -lerl_interface -lei\n", + PortSpecWrite(io_lib:format("ERL_LDFLAGS ?= -L \\"~s\\" -lerl_interface -lei\n", [code:lib_dir(erl_interface, lib)])), [PortSpecWrite(["\n", E, "\n"]) || E <- OsEnv], FilterEnv = fun(Env) -> @@ -4465,7 +4476,7 @@ define dep_autopatch_rebar.erl "%.o: %.C\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", "%.o: %.cc\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", "%.o: %.cpp\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - [[Output, ": ", K, " = ", ShellToMk(V), "\n"] || {K, V} <- lists:reverse(MergeEnv(FilterEnv(Env)))], + [[Output, ": ", K, " += ", ShellToMk(V), "\n"] || {K, V} <- lists:reverse(MergeEnv(FilterEnv(Env)))], Output, ": $$\(foreach ext,.c .C .cc .cpp,", "$$\(patsubst %$$\(ext),%.o,$$\(filter %$$\(ext),$$\(wildcard", Input, "))))\n", "\t$$\(CC) -o $$\@ $$\? $$\(LDFLAGS) $$\(ERL_LDFLAGS) $$\(DRV_LDFLAGS) $$\(EXE_LDFLAGS)", @@ -4588,21 +4599,12 @@ define dep_fetch_cp cp -R $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); endef -define dep_fetch_hex.erl - ssl:start(), - inets:start(), - {ok, {{_, 200, _}, _, Body}} = httpc:request(get, - {"https://s3.amazonaws.com/s3.hex.pm/tarballs/$(1)-$(2).tar", []}, - [], [{body_format, binary}]), - {ok, Files} = erl_tar:extract({binary, Body}, [memory]), - {_, Source} = lists:keyfind("contents.tar.gz", 1, Files), - ok = erl_tar:extract({binary, Source}, [{cwd, "$(call core_native_path,$(DEPS_DIR)/$1)"}, compressed]), - halt() -endef - # Hex only has a package version. No need to look in the Erlang.mk packages. define dep_fetch_hex - $(call erlang,$(call dep_fetch_hex.erl,$(1),$(strip $(word 2,$(dep_$(1)))))); + mkdir -p $(ERLANG_MK_TMP)/hex $(DEPS_DIR)/$1; \ + $(call core_http_get,$(ERLANG_MK_TMP)/hex/$1.tar,\ + https://s3.amazonaws.com/s3.hex.pm/tarballs/$1-$(strip $(word 2,$(dep_$1))).tar); \ + tar -xOf $(ERLANG_MK_TMP)/hex/$1.tar contents.tar.gz | tar -C $(DEPS_DIR)/$1 -xzf -; endef define dep_fetch_fail @@ -4717,7 +4719,7 @@ $(foreach p,$(DEP_PLUGINS),\ $(call core_dep_plugin,$p,$(firstword $(subst /, ,$p))),\ $(call core_dep_plugin,$p/plugins.mk,$p)))) -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. # Configuration. @@ -4734,15 +4736,14 @@ dtl_verbose = $(dtl_verbose_$(V)) # Core targets. -DTL_FILES = $(sort $(call core_find,$(DTL_PATH),*.dtl)) +DTL_PATH := $(abspath $(DTL_PATH)) +DTL_FILES := $(sort $(call core_find,$(DTL_PATH),*.dtl)) ifneq ($(DTL_FILES),) -ifdef DTL_FULL_PATH -BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(subst /,_,$(DTL_FILES:$(DTL_PATH)%=%)))) -else -BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(notdir $(DTL_FILES)))) -endif +DTL_NAMES = $(addsuffix $(DTL_SUFFIX),$(DTL_FILES:$(DTL_PATH)/%.dtl=%)) +DTL_MODULES = $(if $(DTL_FULL_PATH),$(subst /,_,$(DTL_NAMES)),$(notdir $(DTL_NAMES))) +BEAM_FILES += $(addsuffix .beam,$(addprefix ebin/,$(DTL_MODULES))) ifneq ($(words $(DTL_FILES)),0) # Rebuild templates when the Makefile changes. @@ -4762,11 +4763,11 @@ define erlydtl_compile.erl "" -> filename:basename(F, ".dtl"); _ -> - "$(DTL_PATH)" ++ F2 = filename:rootname(F, ".dtl"), + "$(DTL_PATH)/" ++ F2 = filename:rootname(F, ".dtl"), re:replace(F2, "/", "_", [{return, list}, global]) end, Module = list_to_atom(string:to_lower(Module0) ++ "$(DTL_SUFFIX)"), - case erlydtl:compile(F, Module, [$(DTL_OPTS)] ++ [{out_dir, "ebin/"}, return_errors, {doc_root, "templates"}]) of + case erlydtl:compile(F, Module, [$(DTL_OPTS)] ++ [{out_dir, "ebin/"}, return_errors]) of ok -> ok; {ok, _} -> ok end @@ -4776,11 +4777,12 @@ endef ebin/$(PROJECT).app:: $(DTL_FILES) | ebin/ $(if $(strip $?),\ - $(dtl_verbose) $(call erlang,$(call erlydtl_compile.erl,$?),-pa ebin/ $(DEPS_DIR)/erlydtl/ebin/)) + $(dtl_verbose) $(call erlang,$(call erlydtl_compile.erl,$(call core_native_path,$?)),\ + -pa ebin/ $(DEPS_DIR)/erlydtl/ebin/)) endif -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. # Verbosity. @@ -4812,7 +4814,7 @@ ebin/$(PROJECT).app:: $(sort $(call core_find,src/,*.proto)) $(if $(strip $?),$(call compile_proto,$?)) endif -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: clean-app @@ -4881,7 +4883,8 @@ define app_file {id$(comma)$(space)"$(1)"}$(comma)) {modules, [$(call comma_list,$(2))]}, {registered, []}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]} + {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]}, + {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) ]}. endef else @@ -4893,7 +4896,8 @@ define app_file {modules, [$(call comma_list,$(2))]}, {registered, [$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED))]}, {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]}, - {mod, {$(PROJECT_MOD), []}} + {mod, {$(PROJECT_MOD), []}}, + {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) ]}. endef endif @@ -4903,8 +4907,10 @@ app-build: ebin/$(PROJECT).app # Source files. -ERL_FILES = $(sort $(call core_find,src/,*.erl)) -CORE_FILES = $(sort $(call core_find,src/,*.core)) +ALL_SRC_FILES := $(sort $(call core_find,src/,*)) + +ERL_FILES := $(filter %.erl,$(ALL_SRC_FILES)) +CORE_FILES := $(filter %.core,$(ALL_SRC_FILES)) # ASN.1 files. @@ -4937,16 +4943,16 @@ endif # Leex and Yecc files. -XRL_FILES = $(sort $(call core_find,src/,*.xrl)) +XRL_FILES := $(filter %.xrl,$(ALL_SRC_FILES)) XRL_ERL_FILES = $(addprefix src/,$(patsubst %.xrl,%.erl,$(notdir $(XRL_FILES)))) ERL_FILES += $(XRL_ERL_FILES) -YRL_FILES = $(sort $(call core_find,src/,*.yrl)) +YRL_FILES := $(filter %.yrl,$(ALL_SRC_FILES)) YRL_ERL_FILES = $(addprefix src/,$(patsubst %.yrl,%.erl,$(notdir $(YRL_FILES)))) ERL_FILES += $(YRL_ERL_FILES) $(PROJECT).d:: $(XRL_FILES) $(YRL_FILES) - $(if $(strip $?),$(xyrl_verbose) erlc -v -o src/ $?) + $(if $(strip $?),$(xyrl_verbose) erlc -v -o src/ $(YRL_ERLC_OPTS) $?) # Erlang and Core Erlang files. @@ -5035,12 +5041,12 @@ endif ifneq ($(words $(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES)),0) # Rebuild everything when the Makefile changes. $(ERLANG_MK_TMP)/last-makefile-change: $(MAKEFILE_LIST) - @mkdir -p $(ERLANG_MK_TMP) - @if test -f $@; then \ + $(verbose) mkdir -p $(ERLANG_MK_TMP) + $(verbose) if test -f $@; then \ touch $(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES); \ touch -c $(PROJECT).d; \ fi - @touch $@ + $(verbose) touch $@ $(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):: $(ERLANG_MK_TMP)/last-makefile-change ebin/$(PROJECT).app:: $(ERLANG_MK_TMP)/last-makefile-change @@ -5065,7 +5071,7 @@ ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.s $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ $(filter-out $(ERLC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) ifeq ($(wildcard src/$(PROJECT).app.src),) - $(app_verbose) printf "$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES))))" \ + $(app_verbose) printf '$(subst %,%%,$(subst $(newline),\n,$(subst ','\'',$(call app_file,$(GITDESCRIBE),$(MODULES)))))' \ > ebin/$(PROJECT).app else $(verbose) if [ -z "$$(grep -e '^[^%]*{\s*modules\s*,' src/$(PROJECT).app.src)" ]; then \ @@ -5089,6 +5095,7 @@ clean-app: endif +# Copyright (c) 2016, Loïc Hoguin <essen@ninenines.eu> # Copyright (c) 2015, Viktor Söderqvist <viktor@zuiderkwast.se> # This file is part of erlang.mk and subject to the terms of the ISC License. @@ -5109,7 +5116,7 @@ doc-deps: $(ALL_DOC_DEPS_DIRS) $(verbose) for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep; done endif -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: rel-deps @@ -5129,7 +5136,7 @@ rel-deps: $(ALL_REL_DEPS_DIRS) $(verbose) for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep; done endif -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: test-deps test-dir test-build clean-test-dir @@ -5184,7 +5191,7 @@ ifneq ($(wildcard $(TEST_DIR)/*.beam),) endif endif -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: rebar.config @@ -5220,54 +5227,88 @@ $(eval export _compat_rebar_config) rebar.config: $(gen_verbose) echo "$${_compat_rebar_config}" > rebar.config -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. -.PHONY: asciidoc asciidoc-guide asciidoc-manual install-asciidoc distclean-asciidoc +ifeq ($(filter asciideck,$(DEPS) $(DOC_DEPS)),asciideck) -MAN_INSTALL_PATH ?= /usr/local/share/man -MAN_SECTIONS ?= 3 7 +.PHONY: asciidoc asciidoc-guide asciidoc-manual install-asciidoc distclean-asciidoc-guide distclean-asciidoc-manual + +# Core targets. docs:: asciidoc +distclean:: distclean-asciidoc-guide distclean-asciidoc-manual + +# Plugin-specific targets. + asciidoc: asciidoc-guide asciidoc-manual +# User guide. + ifeq ($(wildcard doc/src/guide/book.asciidoc),) asciidoc-guide: else -asciidoc-guide: distclean-asciidoc doc-deps +asciidoc-guide: distclean-asciidoc-guide doc-deps a2x -v -f pdf doc/src/guide/book.asciidoc && mv doc/src/guide/book.pdf doc/guide.pdf a2x -v -f chunked doc/src/guide/book.asciidoc && mv doc/src/guide/book.chunked/ doc/html/ + +distclean-asciidoc-guide: + $(gen_verbose) rm -rf doc/html/ doc/guide.pdf endif -ifeq ($(wildcard doc/src/manual/*.asciidoc),) +# Man pages. + +ASCIIDOC_MANUAL_FILES := $(wildcard doc/src/manual/*.asciidoc) + +ifeq ($(ASCIIDOC_MANUAL_FILES),) asciidoc-manual: else -asciidoc-manual: distclean-asciidoc doc-deps - for f in doc/src/manual/*.asciidoc ; do \ - a2x -v -f manpage $$f ; \ - done - for s in $(MAN_SECTIONS); do \ - mkdir -p doc/man$$s/ ; \ - mv doc/src/manual/*.$$s doc/man$$s/ ; \ - gzip doc/man$$s/*.$$s ; \ - done + +# Configuration. + +MAN_INSTALL_PATH ?= /usr/local/share/man +MAN_SECTIONS ?= 3 7 +MAN_PROJECT ?= $(shell echo $(PROJECT) | sed 's/^./\U&\E/') +MAN_VERSION ?= $(PROJECT_VERSION) + +# Plugin-specific targets. + +define asciidoc2man.erl +try + [begin + ok = asciideck:to_manpage(asciideck:parse_file(F), #{ + compress => gzip, + outdir => filename:dirname(F), + extra2 => "$(MAN_PROJECT) $(MAN_VERSION)", + extra3 => "$(MAN_PROJECT) Function Reference" + }) + end || F <- [$(shell echo $(addprefix $(comma)\",$(addsuffix \",$1)) | sed 's/^.//')]], + halt(0) +catch _:_ -> + halt(1) +end. +endef + +asciidoc-manual:: doc-deps + +asciidoc-manual:: $(ASCIIDOC_MANUAL_FILES) + $(call erlang,$(call asciidoc2man.erl,$?)) + $(foreach s,$(MAN_SECTIONS),mkdir -p doc/man$s/ && mv doc/src/manual/*.$s.gz doc/man$s/;) install-docs:: install-asciidoc install-asciidoc: asciidoc-manual - for s in $(MAN_SECTIONS); do \ - mkdir -p $(MAN_INSTALL_PATH)/man$$s/ ; \ - install -g `id -u` -o `id -g` -m 0644 doc/man$$s/*.gz $(MAN_INSTALL_PATH)/man$$s/ ; \ - done -endif - -distclean:: distclean-asciidoc + $(foreach s,$(MAN_SECTIONS),\ + mkdir -p $(MAN_INSTALL_PATH)/man$s/ && \ + install -g `id -u` -o `id -g` -m 0644 doc/man$s/*.gz $(MAN_INSTALL_PATH)/man$s/;) -distclean-asciidoc: - $(gen_verbose) rm -rf doc/html/ doc/guide.pdf doc/man3/ doc/man7/ +distclean-asciidoc-manual: + $(gen_verbose) rm -rf $(addprefix doc/man,$(MAN_SECTIONS)) +endif +endif -# Copyright (c) 2014-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2014-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: bootstrap bootstrap-lib bootstrap-rel new list-templates @@ -5324,7 +5365,7 @@ ifdef SP define bs_Makefile PROJECT = $p PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 +PROJECT_VERSION = 0.1.0 # Whitespace to be used when creating files from templates. SP = $(SP) @@ -5334,7 +5375,7 @@ else define bs_Makefile PROJECT = $p PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 +PROJECT_VERSION = 0.1.0 endef endif @@ -5342,7 +5383,7 @@ endif define bs_apps_Makefile PROJECT = $p PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 +PROJECT_VERSION = 0.1.0 include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(APPS_DIR)/app)/erlang.mk endef @@ -5362,7 +5403,7 @@ stop(_State) -> endef define bs_relx_config -{release, {$p_release, "1"}, [$p]}. +{release, {$p_release, "1"}, [$p, sasl, runtime_tools]}. {extended_start_script, true}. {sys_config, "rel/sys.config"}. {vm_args, "rel/vm.args"}. @@ -5728,7 +5769,7 @@ endif list-templates: $(verbose) echo Available templates: $(sort $(patsubst tpl_%,%,$(filter tpl_%,$(.VARIABLES)))) -# Copyright (c) 2014-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2014-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: clean-c_src distclean-c_src-env @@ -5963,56 +6004,94 @@ else $(call render_template,bs_erl_nif,src/$n.erl) endif -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. -.PHONY: ci ci-setup distclean-kerl +.PHONY: ci ci-prepare ci-setup distclean-kerl + +CI_OTP ?= +CI_HIPE ?= +CI_ERLLVM ?= + +ifeq ($(CI_VM),native) +ERLC_OPTS += +native +TEST_ERLC_OPTS += +native +else ifeq ($(CI_VM),erllvm) +ERLC_OPTS += +native +'{hipe, [to_llvm]}' +TEST_ERLC_OPTS += +native +'{hipe, [to_llvm]}' +endif + +ifeq ($(strip $(CI_OTP) $(CI_HIPE) $(CI_ERLLVM)),) +ci:: +else + +ifeq ($(strip $(KERL)),) +KERL := $(ERLANG_MK_TMP)/kerl/kerl +endif -KERL ?= $(CURDIR)/kerl export KERL -KERL_URL ?= https://raw.githubusercontent.com/yrashk/kerl/master/kerl +KERL_GIT ?= https://github.com/kerl/kerl +KERL_COMMIT ?= master + +KERL_MAKEFLAGS ?= OTP_GIT ?= https://github.com/erlang/otp CI_INSTALL_DIR ?= $(HOME)/erlang -CI_OTP ?= -ifeq ($(strip $(CI_OTP)),) -ci:: -else -ci:: $(addprefix ci-,$(CI_OTP)) +ci:: $(addprefix ci-,$(CI_OTP) $(addsuffix -native,$(CI_HIPE)) $(addsuffix -erllvm,$(CI_ERLLVM))) -ci-prepare: $(addprefix $(CI_INSTALL_DIR)/,$(CI_OTP)) +ci-prepare: $(addprefix $(CI_INSTALL_DIR)/,$(CI_OTP) $(addsuffix -native,$(CI_HIPE))) ci-setup:: +ci-extra:: + ci_verbose_0 = @echo " CI " $(1); ci_verbose = $(ci_verbose_$(V)) define ci_target -ci-$(1): $(CI_INSTALL_DIR)/$(1) +ci-$1: $(CI_INSTALL_DIR)/$2 + $(verbose) $(MAKE) --no-print-directory clean $(ci_verbose) \ - PATH="$(CI_INSTALL_DIR)/$(1)/bin:$(PATH)" \ - CI_OTP_RELEASE="$(1)" \ - CT_OPTS="-label $(1)" \ - $(MAKE) clean ci-setup tests + PATH="$(CI_INSTALL_DIR)/$2/bin:$(PATH)" \ + CI_OTP_RELEASE="$1" \ + CT_OPTS="-label $1" \ + CI_VM="$3" \ + $(MAKE) ci-setup tests + $(verbose) $(MAKE) --no-print-directory ci-extra endef -$(foreach otp,$(CI_OTP),$(eval $(call ci_target,$(otp)))) +$(foreach otp,$(CI_OTP),$(eval $(call ci_target,$(otp),$(otp),otp))) +$(foreach otp,$(CI_HIPE),$(eval $(call ci_target,$(otp)-native,$(otp)-native,native))) +$(foreach otp,$(CI_ERLLVM),$(eval $(call ci_target,$(otp)-erllvm,$(otp)-native,erllvm))) define ci_otp_target ifeq ($(wildcard $(CI_INSTALL_DIR)/$(1)),) $(CI_INSTALL_DIR)/$(1): $(KERL) - $(KERL) build git $(OTP_GIT) $(1) $(1) + MAKEFLAGS="$(KERL_MAKEFLAGS)" $(KERL) build git $(OTP_GIT) $(1) $(1) $(KERL) install $(1) $(CI_INSTALL_DIR)/$(1) endif endef $(foreach otp,$(CI_OTP),$(eval $(call ci_otp_target,$(otp)))) +define ci_hipe_target +ifeq ($(wildcard $(CI_INSTALL_DIR)/$1-native),) +$(CI_INSTALL_DIR)/$1-native: $(KERL) + KERL_CONFIGURE_OPTIONS=--enable-native-libs \ + MAKEFLAGS="$(KERL_MAKEFLAGS)" $(KERL) build git $(OTP_GIT) $1 $1-native + $(KERL) install $1-native $(CI_INSTALL_DIR)/$1-native +endif +endef + +$(foreach otp,$(sort $(CI_HIPE) $(CI_ERLLLVM)),$(eval $(call ci_hipe_target,$(otp)))) + $(KERL): - $(gen_verbose) $(call core_http_get,$(KERL),$(KERL_URL)) + $(verbose) mkdir -p $(ERLANG_MK_TMP) + $(gen_verbose) git clone --depth 1 $(KERL_GIT) $(ERLANG_MK_TMP)/kerl + $(verbose) cd $(ERLANG_MK_TMP)/kerl && git checkout $(KERL_COMMIT) $(verbose) chmod +x $(KERL) help:: @@ -6029,7 +6108,7 @@ distclean-kerl: $(gen_verbose) rm -rf $(KERL) endif -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: ct apps-ct distclean-ct @@ -6037,11 +6116,13 @@ endif # Configuration. CT_OPTS ?= + ifneq ($(wildcard $(TEST_DIR)),) - CT_SUITES ?= $(sort $(subst _SUITE.erl,,$(notdir $(call core_find,$(TEST_DIR)/,*_SUITE.erl)))) -else - CT_SUITES ?= +ifndef CT_SUITES +CT_SUITES := $(sort $(subst _SUITE.erl,,$(notdir $(call core_find,$(TEST_DIR)/,*_SUITE.erl)))) +endif endif +CT_SUITES ?= # Core targets. @@ -6107,7 +6188,7 @@ $(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test)))) distclean-ct: $(gen_verbose) rm -rf $(CURDIR)/logs/ -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: plt distclean-plt dialyze @@ -6151,7 +6232,8 @@ define filter_opts.erl endef $(DIALYZER_PLT): deps app - $(verbose) dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS) + $(verbose) dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) \ + `test -f $(ERLANG_MK_TMP)/deps.log && cat $(ERLANG_MK_TMP)/deps.log` plt: $(DIALYZER_PLT) @@ -6165,7 +6247,7 @@ dialyze: $(DIALYZER_PLT) endif $(verbose) dialyzer --no_native `$(ERL) -eval "$(subst $(newline),,$(subst ",\",$(call filter_opts.erl)))" -extra $(ERLC_OPTS)` $(DIALYZER_DIRS) $(DIALYZER_OPTS) -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. .PHONY: distclean-edoc edoc @@ -6190,25 +6272,23 @@ edoc: distclean-edoc doc-deps distclean-edoc: $(gen_verbose) rm -f doc/*.css doc/*.html doc/*.png doc/edoc-info -# Copyright (c) 2014 Dave Cottlehuber <dch@skunkwerks.at> +# Copyright (c) 2016, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2014, Dave Cottlehuber <dch@skunkwerks.at> # This file is part of erlang.mk and subject to the terms of the ISC License. -.PHONY: distclean-escript escript +.PHONY: distclean-escript escript escript-zip # Configuration. ESCRIPT_NAME ?= $(PROJECT) ESCRIPT_FILE ?= $(ESCRIPT_NAME) +ESCRIPT_SHEBANG ?= /usr/bin/env escript ESCRIPT_COMMENT ?= This is an -*- erlang -*- file +ESCRIPT_EMU_ARGS ?= -escript main $(ESCRIPT_NAME) -ESCRIPT_BEAMS ?= "ebin/*", "deps/*/ebin/*" -ESCRIPT_SYS_CONFIG ?= "rel/sys.config" -ESCRIPT_EMU_ARGS ?= -pa . \ - -sasl errlog_type error \ - -escript main $(ESCRIPT_NAME) -ESCRIPT_SHEBANG ?= /usr/bin/env escript -ESCRIPT_STATIC ?= "deps/*/priv/**", "priv/**" +ESCRIPT_ZIP ?= 7z a -tzip -mx=9 -mtc=off $(if $(filter-out 0,$(V)),,> /dev/null) +ESCRIPT_ZIP_FILE ?= $(ERLANG_MK_TMP)/escript.zip # Core targets. @@ -6221,44 +6301,28 @@ help:: # Plugin-specific targets. -# Based on https://github.com/synrc/mad/blob/master/src/mad_bundle.erl -# Copyright (c) 2013 Maxim Sokhatsky, Synrc Research Center -# Modified MIT License, https://github.com/synrc/mad/blob/master/LICENSE : -# Software may only be used for the great good and the true happiness of all -# sentient beings. - -define ESCRIPT_RAW -'Read = fun(F) -> {ok, B} = file:read_file(filename:absname(F)), B end,'\ -'Files = fun(L) -> A = lists:concat([filelib:wildcard(X)||X<- L ]),'\ -' [F || F <- A, not filelib:is_dir(F) ] end,'\ -'Squash = fun(L) -> [{filename:basename(F), Read(F) } || F <- L ] end,'\ -'Zip = fun(A, L) -> {ok,{_,Z}} = zip:create(A, L, [{compress,all},memory]), Z end,'\ -'Ez = fun(Escript) ->'\ -' Static = Files([$(ESCRIPT_STATIC)]),'\ -' Beams = Squash(Files([$(ESCRIPT_BEAMS), $(ESCRIPT_SYS_CONFIG)])),'\ -' Archive = Beams ++ [{ "static.gz", Zip("static.gz", Static)}],'\ -' escript:create(Escript, [ $(ESCRIPT_OPTIONS)'\ -' {archive, Archive, [memory]},'\ -' {shebang, "$(ESCRIPT_SHEBANG)"},'\ -' {comment, "$(ESCRIPT_COMMENT)"},'\ -' {emu_args, " $(ESCRIPT_EMU_ARGS)"}'\ -' ]),'\ -' file:change_mode(Escript, 8#755)'\ -'end,'\ -'Ez("$(ESCRIPT_FILE)"),'\ -'halt().' -endef - -ESCRIPT_COMMAND = $(subst ' ',,$(ESCRIPT_RAW)) +escript-zip:: deps app + $(verbose) mkdir -p $(dir $(ESCRIPT_ZIP)) + $(verbose) rm -f $(ESCRIPT_ZIP_FILE) + $(gen_verbose) cd .. && $(ESCRIPT_ZIP) $(ESCRIPT_ZIP_FILE) $(PROJECT)/ebin/* +ifneq ($(DEPS),) + $(verbose) cd $(DEPS_DIR) && $(ESCRIPT_ZIP) $(ESCRIPT_ZIP_FILE) \ + `cat $(ERLANG_MK_TMP)/deps.log | sed 's/^$(subst /,\/,$(DEPS_DIR))\///' | sed 's/$$/\/ebin\/\*/'` +endif -escript:: distclean-escript deps app - $(gen_verbose) $(ERL) -eval $(ESCRIPT_COMMAND) +escript:: escript-zip + $(gen_verbose) printf "%s\n" \ + "#!$(ESCRIPT_SHEBANG)" \ + "%% $(ESCRIPT_COMMENT)" \ + "%%! $(ESCRIPT_EMU_ARGS)" > $(ESCRIPT_FILE) + $(verbose) cat $(ESCRIPT_ZIP_FILE) >> $(ESCRIPT_FILE) + $(verbose) chmod +x $(ESCRIPT_FILE) distclean-escript: $(gen_verbose) rm -f $(ESCRIPT_NAME) +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # Copyright (c) 2014, Enrique Fernandez <enrique.fernandez@erlang-solutions.com> -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> # This file is contributed to erlang.mk and subject to the terms of the ISC License. .PHONY: eunit apps-eunit @@ -6326,14 +6390,14 @@ apps-eunit: endif endif -# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2013-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. -.PHONY: relx-rel distclean-relx-rel distclean-relx run +.PHONY: relx-rel relx-relup distclean-relx-rel run # Configuration. -RELX ?= $(CURDIR)/relx +RELX ?= $(ERLANG_MK_TMP)/relx RELX_CONFIG ?= $(CURDIR)/relx.config RELX_URL ?= https://github.com/erlware/relx/releases/download/v3.19.0/relx @@ -6351,10 +6415,12 @@ endif ifeq ($(IS_DEP),) ifneq ($(wildcard $(RELX_CONFIG)),) rel:: relx-rel + +relup:: relx-relup endif endif -distclean:: distclean-relx-rel distclean-relx +distclean:: distclean-relx-rel # Plugin-specific targets. @@ -6363,14 +6429,14 @@ $(RELX): $(verbose) chmod +x $(RELX) relx-rel: $(RELX) rel-deps app - $(verbose) $(RELX) -c $(RELX_CONFIG) $(RELX_OPTS) + $(verbose) $(RELX) -c $(RELX_CONFIG) $(RELX_OPTS) release tar + +relx-relup: $(RELX) rel-deps app + $(verbose) $(RELX) -c $(RELX_CONFIG) $(RELX_OPTS) release relup tar distclean-relx-rel: $(gen_verbose) rm -rf $(RELX_OUTPUT_DIR) -distclean-relx: - $(gen_verbose) rm -rf $(RELX) - # Run target. ifeq ($(wildcard $(RELX_CONFIG)),) @@ -6379,15 +6445,17 @@ else define get_relx_release.erl {ok, Config} = file:consult("$(RELX_CONFIG)"), - {release, {Name, _}, _} = lists:keyfind(release, 1, Config), - io:format("~s", [Name]), + {release, {Name, Vsn}, _} = lists:keyfind(release, 1, Config), + io:format("~s ~s", [Name, Vsn]), halt(0). endef -RELX_RELEASE = `$(call erlang,$(get_relx_release.erl))` +RELX_REL := $(shell $(call erlang,$(get_relx_release.erl))) +RELX_REL_NAME := $(word 1,$(RELX_REL)) +RELX_REL_VSN := $(word 2,$(RELX_REL)) run: all - $(verbose) $(RELX_OUTPUT_DIR)/$(RELX_RELEASE)/bin/$(RELX_RELEASE) console + $(verbose) $(RELX_OUTPUT_DIR)/$(RELX_REL_NAME)/bin/$(RELX_REL_NAME) console help:: $(verbose) printf "%s\n" "" \ @@ -6396,8 +6464,8 @@ help:: endif +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # Copyright (c) 2014, M Robert Martin <rob@version2beta.com> -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> # This file is contributed to erlang.mk and subject to the terms of the ISC License. .PHONY: shell @@ -6427,7 +6495,7 @@ build-shell-deps: $(ALL_SHELL_DEPS_DIRS) shell: build-shell-deps $(gen_verbose) $(SHELL_ERL) -pa $(SHELL_PATHS) $(SHELL_OPTS) -# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015-2016, Loïc Hoguin <essen@ninenines.eu> # This file is part of erlang.mk and subject to the terms of the ISC License. ifeq ($(filter triq,$(DEPS) $(TEST_DEPS)),triq) @@ -6438,7 +6506,7 @@ ifeq ($(filter triq,$(DEPS) $(TEST_DEPS)),triq) tests:: triq define triq_check.erl - code:add_pathsa(["$(CURDIR)/ebin", "$(DEPS_DIR)/*/ebin"]), + code:add_pathsa(["$(call core_native_path,$(CURDIR)/ebin)", "$(call core_native_path,$(DEPS_DIR)/*/ebin)"]), try case $(1) of all -> [true] =:= lists:usort([triq:check(M) || M <- [$(call comma_list,$(3))]]); @@ -6470,6 +6538,7 @@ triq: test-build endif endif +# Copyright (c) 2016, Loïc Hoguin <essen@ninenines.eu> # Copyright (c) 2015, Erlang Solutions Ltd. # This file is part of erlang.mk and subject to the terms of the ISC License. @@ -6509,7 +6578,8 @@ xref: deps app $(XREFR) distclean-xref: $(gen_verbose) rm -rf $(XREFR) -# Copyright 2015, Viktor Söderqvist <viktor@zuiderkwast.se> +# Copyright (c) 2016, Loïc Hoguin <essen@ninenines.eu> +# Copyright (c) 2015, Viktor Söderqvist <viktor@zuiderkwast.se> # This file is part of erlang.mk and subject to the terms of the ISC License. COVER_REPORT_DIR = cover @@ -6631,6 +6701,53 @@ cover-report: endif endif # ifneq ($(COVER_REPORT_DIR),) +# Copyright (c) 2016, Loïc Hoguin <essen@ninenines.eu> +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: sfx + +ifdef RELX_REL +ifdef SFX + +# Configuration. + +SFX_ARCHIVE ?= $(RELX_OUTPUT_DIR)/$(RELX_REL_NAME)/$(RELX_REL_NAME)-$(RELX_REL_VSN).tar.gz +SFX_OUTPUT_FILE ?= $(RELX_OUTPUT_DIR)/$(RELX_REL_NAME).run + +# Core targets. + +rel:: sfx + +# Plugin-specific targets. + +define sfx_stub +#!/bin/sh + +TMPDIR=`mktemp -d` +ARCHIVE=`awk '/^__ARCHIVE_BELOW__$$/ {print NR + 1; exit 0;}' $$0` +FILENAME=$$(basename $$0) +REL=$${FILENAME%.*} + +tail -n+$$ARCHIVE $$0 | tar -xzf - -C $$TMPDIR + +$$TMPDIR/bin/$$REL console +RET=$$? + +rm -rf $$TMPDIR + +exit $$RET + +__ARCHIVE_BELOW__ +endef + +sfx: + $(call render_template,sfx_stub,$(SFX_OUTPUT_FILE)) + $(gen_verbose) cat $(SFX_ARCHIVE) >> $(SFX_OUTPUT_FILE) + $(verbose) chmod +x $(SFX_OUTPUT_FILE) + +endif +endif + # Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu> # Copyright (c) 2015-2016, Jean-Sébastien Pédron <jean-sebastien@rabbitmq.com> # This file is part of erlang.mk and subject to the terms of the ISC License. diff --git a/include/rabbit_cli.hrl b/include/rabbit_cli.hrl index b1cf41261f..53be9fcda0 100644 --- a/include/rabbit_cli.hrl +++ b/include/rabbit_cli.hrl @@ -31,6 +31,12 @@ -define(ONLINE_OPT, "--online"). -define(LOCAL_OPT, "--local"). +-define(DECODE_OPT, "--decode"). +-define(CIPHER_OPT, "--cipher"). +-define(HASH_OPT, "--hash"). +-define(ITERATIONS_OPT, "--iterations"). +-define(LIST_CIPHERS_OPT, "--list-ciphers"). +-define(LIST_HASHES_OPT, "--list-hashes"). -define(NODE_DEF(Node), {?NODE_OPT, {option, Node}}). -define(QUIET_DEF, {?QUIET_OPT, flag}). @@ -48,6 +54,13 @@ -define(OFFLINE_DEF, {?OFFLINE_OPT, flag}). -define(ONLINE_DEF, {?ONLINE_OPT, flag}). -define(LOCAL_DEF, {?LOCAL_OPT, flag}). +-define(DECODE_DEF, {?DECODE_OPT, flag}). +-define(CIPHER_DEF, {?CIPHER_OPT, {option, atom_to_list(rabbit_pbe:default_cipher())}}). +-define(HASH_DEF, {?HASH_OPT, {option, atom_to_list(rabbit_pbe:default_hash())}}). +-define(ITERATIONS_DEF, {?ITERATIONS_OPT, {option, integer_to_list(rabbit_pbe:default_iterations())}}). +-define(LIST_CIPHERS_DEF, {?LIST_CIPHERS_OPT, flag}). +-define(LIST_HASHES_DEF, {?LIST_HASHES_OPT, flag}). + %% Subset of standartized exit codes from sysexits.h, see %% https://github.com/rabbitmq/rabbitmq-server/issues/396 for discussion. diff --git a/rabbitmq-components.mk b/rabbitmq-components.mk index 05986d82ce..1ce965cc83 100644 --- a/rabbitmq-components.mk +++ b/rabbitmq-components.mk @@ -26,8 +26,10 @@ dep_rabbitmq_auth_backend_ldap = git_rmq rabbitmq-auth-backend-ldap $(cur dep_rabbitmq_auth_mechanism_ssl = git_rmq rabbitmq-auth-mechanism-ssl $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_boot_steps_visualiser = git_rmq rabbitmq-boot-steps-visualiser $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_clusterer = git_rmq rabbitmq-clusterer $(current_rmq_ref) $(base_rmq_ref) master +dep_rabbitmq_cli = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_codegen = git_rmq rabbitmq-codegen $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_consistent_hash_exchange = git_rmq rabbitmq-consistent-hash-exchange $(current_rmq_ref) $(base_rmq_ref) master +dep_rabbitmq_ct_client_helpers = git_rmq rabbitmq-ct-client-helpers $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_ct_helpers = git_rmq rabbitmq-ct-helpers $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_delayed_message_exchange = git_rmq rabbitmq-delayed-message-exchange $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_dotnet_client = git_rmq rabbitmq-dotnet-client $(current_rmq_ref) $(base_rmq_ref) master @@ -88,8 +90,10 @@ RABBITMQ_COMPONENTS = amqp_client \ rabbitmq_auth_mechanism_ssl \ rabbitmq_boot_steps_visualiser \ rabbitmq_clusterer \ + rabbitmq_cli \ rabbitmq_codegen \ rabbitmq_consistent_hash_exchange \ + rabbitmq_ct_client_helpers \ rabbitmq_ct_helpers \ rabbitmq_delayed_message_exchange \ rabbitmq_dotnet_client \ diff --git a/src/gm.erl b/src/gm.erl index 74e19ee6fd..41aa01f04d 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -1175,11 +1175,20 @@ record_new_member_in_group(NewMember, Left, GroupName, TxnFun) -> try Group = #gm_group { members = Members, version = Ver } = check_membership(Left, read_group(GroupName)), - {Prefix, [Left | Suffix]} = - lists:splitwith(fun (M) -> M =/= Left end, Members), - write_group(Group #gm_group { - members = Prefix ++ [Left, NewMember | Suffix], - version = Ver + 1 }) + case lists:member(NewMember, Members) of + true -> + %% This avois duplicates during partial partitions, + %% as inconsistent views might happen during them + rabbit_log:warning("(~p) GM avoiding duplicate of ~p", + [self(), NewMember]), + Group; + false -> + {Prefix, [Left | Suffix]} = + lists:splitwith(fun (M) -> M =/= Left end, Members), + write_group(Group #gm_group { + members = Prefix ++ [Left, NewMember | Suffix], + version = Ver + 1 }) + end catch lost_membership -> %% The transaction must not be abruptly crashed, but diff --git a/src/pg2_fixed.erl b/src/pg2_fixed.erl deleted file mode 100644 index 222a0bc849..0000000000 --- a/src/pg2_fixed.erl +++ /dev/null @@ -1,399 +0,0 @@ -%% This is the version of pg2 from R14B02, which contains the fix -%% described at -%% http://erlang.2086793.n4.nabble.com/pg2-still-busted-in-R13B04-td2230601.html. -%% The changes are a search-and-replace to rename the module and avoid -%% clashes with other versions of pg2, and also a simple rewrite of -%% "andalso" and "orelse" expressions to case statements where the second -%% operand is not a boolean since R12B does not allow this. - -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% --module(pg2_fixed). - --export([create/1, delete/1, join/2, leave/2]). --export([get_members/1, get_local_members/1]). --export([get_closest_pid/1, which_groups/0]). --export([start/0,start_link/0,init/1,handle_call/3,handle_cast/2,handle_info/2, - terminate/2]). - -%%% As of R13B03 monitors are used instead of links. - -%%% -%%% Exported functions -%%% - --spec start_link() -> {'ok', pid()} | {'error', term()}. - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec start() -> {'ok', pid()} | {'error', term()}. - -start() -> - ensure_started(). - --spec create(term()) -> 'ok'. - -create(Name) -> - _ = ensure_started(), - case ets:member(pg2_fixed_table, {group, Name}) of - false -> - global:trans({{?MODULE, Name}, self()}, - fun() -> - gen_server:multi_call(?MODULE, {create, Name}) - end), - ok; - true -> - ok - end. - --type name() :: term(). - --spec delete(name()) -> 'ok'. - -delete(Name) -> - _ = ensure_started(), - global:trans({{?MODULE, Name}, self()}, - fun() -> - gen_server:multi_call(?MODULE, {delete, Name}) - end), - ok. - --spec join(name(), pid()) -> 'ok' | {'error', {'no_such_group', term()}}. - -join(Name, Pid) when is_pid(Pid) -> - _ = ensure_started(), - case ets:member(pg2_fixed_table, {group, Name}) of - false -> - {error, {no_such_group, Name}}; - true -> - global:trans({{?MODULE, Name}, self()}, - fun() -> - gen_server:multi_call(?MODULE, - {join, Name, Pid}) - end), - ok - end. - --spec leave(name(), pid()) -> 'ok' | {'error', {'no_such_group', name()}}. - -leave(Name, Pid) when is_pid(Pid) -> - _ = ensure_started(), - case ets:member(pg2_fixed_table, {group, Name}) of - false -> - {error, {no_such_group, Name}}; - true -> - global:trans({{?MODULE, Name}, self()}, - fun() -> - gen_server:multi_call(?MODULE, - {leave, Name, Pid}) - end), - ok - end. - --type get_members_ret() :: [pid()] | {'error', {'no_such_group', name()}}. - --spec get_members(name()) -> get_members_ret(). - -get_members(Name) -> - _ = ensure_started(), - case ets:member(pg2_fixed_table, {group, Name}) of - true -> - group_members(Name); - false -> - {error, {no_such_group, Name}} - end. - --spec get_local_members(name()) -> get_members_ret(). - -get_local_members(Name) -> - _ = ensure_started(), - case ets:member(pg2_fixed_table, {group, Name}) of - true -> - local_group_members(Name); - false -> - {error, {no_such_group, Name}} - end. - --spec which_groups() -> [name()]. - -which_groups() -> - _ = ensure_started(), - all_groups(). - --type gcp_error_reason() :: {'no_process', term()} | {'no_such_group', term()}. - --spec get_closest_pid(term()) -> pid() | {'error', gcp_error_reason()}. - -get_closest_pid(Name) -> - case get_local_members(Name) of - [Pid] -> - Pid; - [] -> - case get_members(Name) of - [] -> {error, {no_process, Name}}; - Members -> - X = time_compat:erlang_system_time(micro_seconds), - lists:nth((X rem length(Members))+1, Members) - end; - Members when is_list(Members) -> - X = time_compat:erlang_system_time(micro_seconds), - lists:nth((X rem length(Members))+1, Members); - Else -> - Else - end. - -%%% -%%% Callback functions from gen_server -%%% - --record(state, {}). - --spec init([]) -> {'ok', #state{}}. - -init([]) -> - Ns = nodes(), - _ = net_kernel:monitor_nodes(true), - lists:foreach(fun(N) -> - {?MODULE, N} ! {new_pg2_fixed, node()}, - self() ! {nodeup, N} - end, Ns), - pg2_fixed_table = ets:new(pg2_fixed_table, [ordered_set, protected, named_table]), - {ok, #state{}}. - --type call() :: {'create', name()} - | {'delete', name()} - | {'join', name(), pid()} - | {'leave', name(), pid()}. - --spec handle_call(call(), _, #state{}) -> - {'reply', 'ok', #state{}}. - -handle_call({create, Name}, _From, S) -> - assure_group(Name), - {reply, ok, S}; -handle_call({join, Name, Pid}, _From, S) -> - case ets:member(pg2_fixed_table, {group, Name}) of - true -> _ = join_group(Name, Pid), - ok; - _ -> ok - end, - {reply, ok, S}; -handle_call({leave, Name, Pid}, _From, S) -> - case ets:member(pg2_fixed_table, {group, Name}) of - true -> leave_group(Name, Pid); - _ -> ok - end, - {reply, ok, S}; -handle_call({delete, Name}, _From, S) -> - delete_group(Name), - {reply, ok, S}; -handle_call(Request, From, S) -> - error_logger:warning_msg("The pg2_fixed server received an unexpected message:\n" - "handle_call(~p, ~p, _)\n", - [Request, From]), - {noreply, S}. - --type all_members() :: [[name(),...]]. --type cast() :: {'exchange', node(), all_members()} - | {'del_member', name(), pid()}. - --spec handle_cast(cast(), #state{}) -> {'noreply', #state{}}. - -handle_cast({exchange, _Node, List}, S) -> - store(List), - {noreply, S}; -handle_cast(_, S) -> - %% Ignore {del_member, Name, Pid}. - {noreply, S}. - --spec handle_info(tuple(), #state{}) -> {'noreply', #state{}}. - -handle_info({'DOWN', MonitorRef, process, _Pid, _Info}, S) -> - member_died(MonitorRef), - {noreply, S}; -handle_info({nodeup, Node}, S) -> - gen_server:cast({?MODULE, Node}, {exchange, node(), all_members()}), - {noreply, S}; -handle_info({new_pg2_fixed, Node}, S) -> - gen_server:cast({?MODULE, Node}, {exchange, node(), all_members()}), - {noreply, S}; -handle_info(_, S) -> - {noreply, S}. - --spec terminate(term(), #state{}) -> 'ok'. - -terminate(_Reason, _S) -> - true = ets:delete(pg2_fixed_table), - ok. - -%%% -%%% Local functions -%%% - -%%% One ETS table, pg2_fixed_table, is used for bookkeeping. The type of the -%%% table is ordered_set, and the fast matching of partially -%%% instantiated keys is used extensively. -%%% -%%% {{group, Name}} -%%% Process group Name. -%%% {{ref, Pid}, RPid, MonitorRef, Counter} -%%% {{ref, MonitorRef}, Pid} -%%% Each process has one monitor. Sometimes a process is spawned to -%%% monitor the pid (RPid). Counter is incremented when the Pid joins -%%% some group. -%%% {{member, Name, Pid}, GroupCounter} -%%% {{local_member, Name, Pid}} -%%% Pid is a member of group Name, GroupCounter is incremented when the -%%% Pid joins the group Name. -%%% {{pid, Pid, Name}} -%%% Pid is a member of group Name. - -store(List) -> - _ = [assure_group(Name) - andalso - [join_group(Name, P) || P <- Members -- group_members(Name)] || - [Name, Members] <- List], - ok. - -assure_group(Name) -> - Key = {group, Name}, - ets:member(pg2_fixed_table, Key) orelse true =:= ets:insert(pg2_fixed_table, {Key}). - -delete_group(Name) -> - _ = [leave_group(Name, Pid) || Pid <- group_members(Name)], - true = ets:delete(pg2_fixed_table, {group, Name}), - ok. - -member_died(Ref) -> - [{{ref, Ref}, Pid}] = ets:lookup(pg2_fixed_table, {ref, Ref}), - Names = member_groups(Pid), - _ = [leave_group(Name, P) || - Name <- Names, - P <- member_in_group(Pid, Name)], - %% Kept for backward compatibility with links. Can be removed, eventually. - _ = [gen_server:abcast(nodes(), ?MODULE, {del_member, Name, Pid}) || - Name <- Names], - ok. - -join_group(Name, Pid) -> - Ref_Pid = {ref, Pid}, - try _ = ets:update_counter(pg2_fixed_table, Ref_Pid, {4, +1}) - catch _:_ -> - {RPid, Ref} = do_monitor(Pid), - true = ets:insert(pg2_fixed_table, {Ref_Pid, RPid, Ref, 1}), - true = ets:insert(pg2_fixed_table, {{ref, Ref}, Pid}) - end, - Member_Name_Pid = {member, Name, Pid}, - try _ = ets:update_counter(pg2_fixed_table, Member_Name_Pid, {2, +1, 1, 1}) - catch _:_ -> - true = ets:insert(pg2_fixed_table, {Member_Name_Pid, 1}), - _ = [ets:insert(pg2_fixed_table, {{local_member, Name, Pid}}) || - node(Pid) =:= node()], - true = ets:insert(pg2_fixed_table, {{pid, Pid, Name}}) - end. - -leave_group(Name, Pid) -> - Member_Name_Pid = {member, Name, Pid}, - try ets:update_counter(pg2_fixed_table, Member_Name_Pid, {2, -1, 0, 0}) of - N -> - if - N =:= 0 -> - true = ets:delete(pg2_fixed_table, {pid, Pid, Name}), - _ = [ets:delete(pg2_fixed_table, {local_member, Name, Pid}) || - node(Pid) =:= node()], - true = ets:delete(pg2_fixed_table, Member_Name_Pid); - true -> - ok - end, - Ref_Pid = {ref, Pid}, - case ets:update_counter(pg2_fixed_table, Ref_Pid, {4, -1}) of - 0 -> - [{Ref_Pid,RPid,Ref,0}] = ets:lookup(pg2_fixed_table, Ref_Pid), - true = ets:delete(pg2_fixed_table, {ref, Ref}), - true = ets:delete(pg2_fixed_table, Ref_Pid), - true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid); - _ -> - ok - end - catch _:_ -> - ok - end. - -all_members() -> - [[G, group_members(G)] || G <- all_groups()]. - -group_members(Name) -> - [P || - [P, N] <- ets:match(pg2_fixed_table, {{member, Name, '$1'},'$2'}), - _ <- lists:seq(1, N)]. - -local_group_members(Name) -> - [P || - [Pid] <- ets:match(pg2_fixed_table, {{local_member, Name, '$1'}}), - P <- member_in_group(Pid, Name)]. - -member_in_group(Pid, Name) -> - case ets:lookup(pg2_fixed_table, {member, Name, Pid}) of - [] -> []; - [{{member, Name, Pid}, N}] -> - lists:duplicate(N, Pid) - end. - -member_groups(Pid) -> - [Name || [Name] <- ets:match(pg2_fixed_table, {{pid, Pid, '$1'}})]. - -all_groups() -> - [N || [N] <- ets:match(pg2_fixed_table, {{group,'$1'}})]. - -ensure_started() -> - case whereis(?MODULE) of - undefined -> - C = {pg2_fixed, {?MODULE, start_link, []}, permanent, - 1000, worker, [?MODULE]}, - supervisor:start_child(kernel_safe_sup, C); - Pg2_FixedPid -> - {ok, Pg2_FixedPid} - end. - - -kill_monitor_proc(RPid, Pid) -> - case RPid of - Pid -> ok; - _ -> exit(RPid, kill) - end. - -%% When/if erlang:monitor() returns before trying to connect to the -%% other node this function can be removed. -do_monitor(Pid) -> - case (node(Pid) =:= node()) orelse lists:member(node(Pid), nodes()) of - true -> - %% Assume the node is still up - {Pid, erlang:monitor(process, Pid)}; - false -> - F = fun() -> - Ref = erlang:monitor(process, Pid), - receive - {'DOWN', Ref, process, Pid, _Info} -> - exit(normal) - end - end, - erlang:spawn_monitor(F) - end. diff --git a/src/rabbit.app.src b/src/rabbit.app.src index 872336bd8e..dd38ad4072 100644 --- a/src/rabbit.app.src +++ b/src/rabbit.app.src @@ -99,5 +99,13 @@ {credit_flow_default_credit, {200, 100}}, %% see rabbitmq-server#248 %% and rabbitmq-server#667 - {channel_operation_timeout, 15000} + {channel_operation_timeout, 15000}, + {config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000}, + {passphrase, undefined} + ]}, + %% rabbitmq-server-973 + {lazy_queue_explicit_gc_run_operation_threshold, 250} ]}]}. diff --git a/src/rabbit.erl b/src/rabbit.erl index c9a008c446..f0d66f4110 100644 --- a/src/rabbit.erl +++ b/src/rabbit.erl @@ -24,7 +24,7 @@ start_fhc/0]). -export([start/2, stop/1, prep_stop/1]). -export([start_apps/1, stop_apps/1]). --export([log_location/1, config_files/0]). %% for testing and mgmt-agent +-export([log_location/1, config_files/0, decrypt_config/2]). %% for testing and mgmt-agent %%--------------------------------------------------------------------------- %% Boot steps. @@ -449,6 +449,38 @@ stop_and_halt() -> start_apps(Apps) -> app_utils:load_applications(Apps), + + ConfigEntryDecoder = case application:get_env(rabbit, config_entry_decoder) of + undefined -> + []; + {ok, Val} -> + Val + end, + PassPhrase = case proplists:get_value(passphrase, ConfigEntryDecoder) of + prompt -> + IoDevice = get_input_iodevice(), + io:setopts(IoDevice, [{echo, false}]), + PP = lists:droplast(io:get_line(IoDevice, + "\nPlease enter the passphrase to unlock encrypted " + "configuration entries.\n\nPassphrase: ")), + io:setopts(IoDevice, [{echo, true}]), + io:format(IoDevice, "~n", []), + PP; + {file, Filename} -> + {ok, File} = file:read_file(Filename), + [PP|_] = binary:split(File, [<<"\r\n">>, <<"\n">>]), + PP; + PP -> + PP + end, + Algo = { + proplists:get_value(cipher, ConfigEntryDecoder, rabbit_pbe:default_cipher()), + proplists:get_value(hash, ConfigEntryDecoder, rabbit_pbe:default_hash()), + proplists:get_value(iterations, ConfigEntryDecoder, rabbit_pbe:default_iterations()), + PassPhrase + }, + decrypt_config(Apps, Algo), + OrderedApps = app_utils:app_dependency_order(Apps, false), case lists:member(rabbit, Apps) of false -> rabbit_boot_steps:run_boot_steps(Apps); %% plugin activation @@ -457,6 +489,78 @@ start_apps(Apps) -> ok = app_utils:start_applications(OrderedApps, handle_app_error(could_not_start)). +%% This function retrieves the correct IoDevice for requesting +%% input. The problem with using the default IoDevice is that +%% the Erlang shell prevents us from getting the input. +%% +%% Instead we therefore look for the io process used by the +%% shell and if it can't be found (because the shell is not +%% started e.g with -noshell) we use the 'user' process. +%% +%% This function will not work when either -oldshell or -noinput +%% options are passed to erl. +get_input_iodevice() -> + case whereis(user) of + undefined -> user; + User -> + case group:interfaces(User) of + [] -> + user; + [{user_drv, Drv}] -> + case user_drv:interfaces(Drv) of + [] -> + user; + [{current_group, IoDevice}] -> + IoDevice + end + end + end. + +decrypt_config([], _) -> + ok; +decrypt_config([App|Apps], Algo) -> + decrypt_app(App, application:get_all_env(App), Algo), + decrypt_config(Apps, Algo). + +decrypt_app(_, [], _) -> + ok; +decrypt_app(App, [{Key, Value}|Tail], Algo) -> + try begin + case decrypt(Value, Algo) of + Value -> + ok; + NewValue -> + application:set_env(App, Key, NewValue) + end + end + catch + exit:{bad_configuration, config_entry_decoder} -> + exit({bad_configuration, config_entry_decoder}); + _:Msg -> + rabbit_log:info("Error while decrypting key '~p'. Please check encrypted value, passphrase, and encryption configuration~n", [Key]), + exit({decryption_error, {key, Key}, Msg}) + end, + decrypt_app(App, Tail, Algo). + +decrypt({encrypted, _}, {_, _, _, undefined}) -> + exit({bad_configuration, config_entry_decoder}); +decrypt({encrypted, EncValue}, {Cipher, Hash, Iterations, Password}) -> + rabbit_pbe:decrypt_term(Cipher, Hash, Iterations, Password, EncValue); +decrypt(List, Algo) when is_list(List) -> + decrypt_list(List, Algo, []); +decrypt(Value, _) -> + Value. + +%% We make no distinction between strings and other lists. +%% When we receive a string, we loop through each element +%% and ultimately return the string unmodified, as intended. +decrypt_list([], _, Acc) -> + lists:reverse(Acc); +decrypt_list([{Key, Value}|Tail], Algo, Acc) when Key =/= encrypted -> + decrypt_list(Tail, Algo, [{Key, decrypt(Value, Algo)}|Acc]); +decrypt_list([Value|Tail], Algo, Acc) -> + decrypt_list(Tail, Algo, [decrypt(Value, Algo)|Acc]). + stop_apps(Apps) -> ok = app_utils:stop_applications( Apps, handle_app_error(error_during_shutdown)), diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl index 4f8581f78a..b7df5a9a19 100644 --- a/src/rabbit_amqqueue_process.erl +++ b/src/rabbit_amqqueue_process.erl @@ -1310,7 +1310,13 @@ handle_cast(policy_changed, State = #q{q = #amqqueue{name = Name}}) -> %% This also has the side effect of waking us up so we emit a %% stats event - so event consumers see the changed policy. {ok, Q} = rabbit_amqqueue:lookup(Name), - noreply(process_args_policy(State#q{q = Q})). + noreply(process_args_policy(State#q{q = Q})); + +handle_cast({sync_start, _, _}, State = #q{q = #amqqueue{name = Name}}) -> + %% Only a slave should receive this, it means we are a duplicated master + rabbit_mirror_queue_misc:log_warning( + Name, "Stopping after receiving sync_start from another master", []), + stop(State). handle_info({maybe_expire, Vsn}, State = #q{args_policy_version = Vsn}) -> case is_unused(State) of diff --git a/src/rabbit_control_main.erl b/src/rabbit_control_main.erl index 92898c2a2c..8c245892b7 100644 --- a/src/rabbit_control_main.erl +++ b/src/rabbit_control_main.erl @@ -92,7 +92,8 @@ {trace_off, [?VHOST_DEF]}, set_vm_memory_high_watermark, set_disk_free_limit, - help + help, + {encode, [?DECODE_DEF, ?CIPHER_DEF, ?HASH_DEF, ?ITERATIONS_DEF, ?LIST_CIPHERS_DEF, ?LIST_HASHES_DEF]} ]). -define(GLOBAL_QUERIES, @@ -114,7 +115,7 @@ [stop, stop_app, start_app, wait, reset, force_reset, rotate_logs, join_cluster, change_cluster_node_type, update_cluster_nodes, forget_cluster_node, rename_cluster_node, cluster_status, status, - environment, eval, force_boot, help, hipe_compile]). + environment, eval, force_boot, help, hipe_compile, encode]). %% [Command | {Command, DefaultTimeoutInMilliSeconds}] -define(COMMANDS_WITH_TIMEOUT, @@ -579,6 +580,17 @@ action(eval, Node, [Expr], _Opts, _Inform) -> action(help, _Node, _Args, _Opts, _Inform) -> io:format("~s", [rabbit_ctl_usage:usage()]); +action(encode, _Node, Args, Opts, _Inform) -> + ListCiphers = lists:member({?LIST_CIPHERS_OPT, true}, Opts), + ListHashes = lists:member({?LIST_HASHES_OPT, true}, Opts), + Decode = lists:member({?DECODE_OPT, true}, Opts), + Cipher = list_to_atom(proplists:get_value(?CIPHER_OPT, Opts)), + Hash = list_to_atom(proplists:get_value(?HASH_OPT, Opts)), + Iterations = list_to_integer(proplists:get_value(?ITERATIONS_OPT, Opts)), + + {_, Msg} = rabbit_control_pbe:encode(ListCiphers, ListHashes, Decode, Cipher, Hash, Iterations, Args), + io:format(Msg ++ "~n"); + action(Command, Node, Args, Opts, Inform) -> %% For backward compatibility, run commands accepting a timeout with %% the default timeout. diff --git a/src/rabbit_control_pbe.erl b/src/rabbit_control_pbe.erl new file mode 100644 index 0000000000..2fa2c90a6e --- /dev/null +++ b/src/rabbit_control_pbe.erl @@ -0,0 +1,79 @@ +% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_control_pbe). + +-export([encode/7]). + +% for testing purposes +-export([evaluate_input_as_term/1]). + +encode(ListCiphers, _ListHashes, _Decode, _Cipher, _Hash, _Iterations, _Args) when ListCiphers -> + {ok, io_lib:format("~p", [rabbit_pbe:supported_ciphers()])}; + +encode(_ListCiphers, ListHashes, _Decode, _Cipher, _Hash, _Iterations, _Args) when ListHashes -> + {ok, io_lib:format("~p", [rabbit_pbe:supported_hashes()])}; + +encode(_ListCiphers, _ListHashes, Decode, Cipher, Hash, Iterations, Args) -> + CipherExists = lists:member(Cipher, rabbit_pbe:supported_ciphers()), + HashExists = lists:member(Hash, rabbit_pbe:supported_hashes()), + encode_encrypt_decrypt(CipherExists, HashExists, Decode, Cipher, Hash, Iterations, Args). + +encode_encrypt_decrypt(CipherExists, _HashExists, _Decode, _Cipher, _Hash, _Iterations, _Args) when CipherExists =:= false -> + {error, io_lib:format("The requested cipher is not supported", [])}; + +encode_encrypt_decrypt(_CipherExists, HashExists, _Decode, _Cipher, _Hash, _Iterations, _Args) when HashExists =:= false -> + {error, io_lib:format("The requested hash is not supported", [])}; + +encode_encrypt_decrypt(_CipherExists, _HashExists, _Decode, _Cipher, _Hash, Iterations, _Args) when Iterations =< 0 -> + {error, io_lib:format("The requested number of iterations is incorrect", [])}; + +encode_encrypt_decrypt(_CipherExists, _HashExists, Decode, Cipher, Hash, Iterations, Args) when length(Args) == 2, Decode =:= false -> + [Value, PassPhrase] = Args, + try begin + TermValue = evaluate_input_as_term(Value), + Result = rabbit_pbe:encrypt_term(Cipher, Hash, Iterations, list_to_binary(PassPhrase), TermValue), + {ok, io_lib:format("~p", [{encrypted, Result}])} + end + catch + _:Msg -> {error, io_lib:format("Error during cipher operation: ~p", [Msg])} + end; + +encode_encrypt_decrypt(_CipherExists, _HashExists, Decode, Cipher, Hash, Iterations, Args) when length(Args) == 2, Decode -> + [Value, PassPhrase] = Args, + try begin + TermValue = evaluate_input_as_term(Value), + TermToDecrypt = case TermValue of + {encrypted, EncryptedTerm} -> + EncryptedTerm; + _ -> + TermValue + end, + Result = rabbit_pbe:decrypt_term(Cipher, Hash, Iterations, list_to_binary(PassPhrase), TermToDecrypt), + {ok, io_lib:format("~p", [Result])} + end + catch + _:Msg -> {error, io_lib:format("Error during cipher operation: ~p", [Msg])} + end; + +encode_encrypt_decrypt(_CipherExists, _HashExists, _Decode, _Cipher, _Hash, _Iterations, _Args) -> + {error, io_lib:format("Please provide a value to encode/decode and a passphrase", [])}. + +evaluate_input_as_term(Input) -> + {ok,Tokens,_EndLine} = erl_scan:string(Input ++ "."), + {ok,AbsForm} = erl_parse:parse_exprs(Tokens), + {value,TermValue,_Bs} = erl_eval:exprs(AbsForm, erl_eval:new_bindings()), + TermValue. diff --git a/src/rabbit_mirror_queue_mode_nodes.erl b/src/rabbit_mirror_queue_mode_nodes.erl index e63f340373..31c55722a5 100644 --- a/src/rabbit_mirror_queue_mode_nodes.erl +++ b/src/rabbit_mirror_queue_mode_nodes.erl @@ -32,29 +32,37 @@ description() -> [{description, <<"Mirror queue to specified nodes">>}]. -suggested_queue_nodes(Nodes0, MNode, _SNodes, SSNodes, Poss) -> - Nodes1 = [list_to_atom(binary_to_list(Node)) || Node <- Nodes0], +suggested_queue_nodes(PolicyNodes0, CurrentMaster, _SNodes, SSNodes, NodesRunningRabbitMQ) -> + PolicyNodes1 = [list_to_atom(binary_to_list(Node)) || Node <- PolicyNodes0], %% If the current master is not in the nodes specified, then what we want %% to do depends on whether there are any synchronised slaves. If there %% are then we can just kill the current master - the admin has asked for %% a migration and we should give it to them. If there are not however %% then we must keep the master around so as not to lose messages. - Nodes = case SSNodes of - [] -> lists:usort([MNode | Nodes1]); - _ -> Nodes1 - end, - Unavailable = Nodes -- Poss, - Available = Nodes -- Unavailable, - case Available of + + PolicyNodes = case SSNodes of + [] -> lists:usort([CurrentMaster | PolicyNodes1]); + _ -> PolicyNodes1 + end, + Unavailable = PolicyNodes -- NodesRunningRabbitMQ, + AvailablePolicyNodes = PolicyNodes -- Unavailable, + case AvailablePolicyNodes of [] -> %% We have never heard of anything? Not much we can do but %% keep the master alive. - {MNode, []}; - _ -> case lists:member(MNode, Available) of - true -> {MNode, Available -- [MNode]}; + {CurrentMaster, []}; + _ -> case lists:member(CurrentMaster, AvailablePolicyNodes) of + true -> {CurrentMaster, + AvailablePolicyNodes -- [CurrentMaster]}; false -> %% Make sure the new master is synced! In order to %% get here SSNodes must not be empty. - [NewMNode | _] = SSNodes, - {NewMNode, Available -- [NewMNode]} + SyncPolicyNodes = [Node || + Node <- AvailablePolicyNodes, + lists:member(Node, SSNodes)], + NewMaster = case SyncPolicyNodes of + [Node | _] -> Node; + [] -> erlang:hd(SSNodes) + end, + {NewMaster, AvailablePolicyNodes -- [NewMaster]} end end. diff --git a/src/rabbit_mirror_queue_slave.erl b/src/rabbit_mirror_queue_slave.erl index 4770018f9e..6017e5a028 100644 --- a/src/rabbit_mirror_queue_slave.erl +++ b/src/rabbit_mirror_queue_slave.erl @@ -314,7 +314,12 @@ handle_cast({set_ram_duration_target, Duration}, State = #state { backing_queue = BQ, backing_queue_state = BQS }) -> BQS1 = BQ:set_ram_duration_target(Duration, BQS), - noreply(State #state { backing_queue_state = BQS1 }). + noreply(State #state { backing_queue_state = BQS1 }); + +handle_cast(policy_changed, State) -> + %% During partial partitions, we might end up receiving messages expected by a master + %% Ignore them + noreply(State). handle_info(update_ram_duration, State = #state{backing_queue = BQ, backing_queue_state = BQS}) -> diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index 297df086ad..dd92256146 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -306,7 +306,11 @@ io_batch_size, %% default queue or lazy queue - mode + mode, + %% number of reduce_memory_usage executions, once it + %% reaches a threshold the queue will manually trigger a runtime GC + %% see: maybe_execute_gc/1 + memory_reduction_run_count }). -record(rates, { in, out, ack_in, ack_out, timestamp }). @@ -402,7 +406,8 @@ disk_write_count :: non_neg_integer(), io_batch_size :: pos_integer(), - mode :: 'default' | 'lazy' }. + mode :: 'default' | 'lazy', + memory_reduction_run_count :: non_neg_integer()}. %% Duplicated from rabbit_backing_queue -spec ack([ack()], state()) -> {[rabbit_guid:guid()], state()}. @@ -427,6 +432,21 @@ %% rabbit_amqqueue_process need fairly fresh rates. -define(MSGS_PER_RATE_CALC, 100). + +%% we define the garbage collector threshold +%% it needs to tune the GC calls inside `reduce_memory_use` +%% see: rabbitmq-server-973 and `maybe_execute_gc` function +-define(DEFAULT_EXPLICIT_GC_RUN_OP_THRESHOLD, 250). +-define(EXPLICIT_GC_RUN_OP_THRESHOLD, + case get(explicit_gc_run_operation_threshold) of + undefined -> + Val = rabbit_misc:get_env(rabbit, lazy_queue_explicit_gc_run_operation_threshold, + ?DEFAULT_EXPLICIT_GC_RUN_OP_THRESHOLD), + put(explicit_gc_run_operation_threshold, Val), + Val; + Val -> Val + end). + %%---------------------------------------------------------------------------- %% Public API %%---------------------------------------------------------------------------- @@ -633,25 +653,28 @@ ack([], State) -> %% optimisation: this head is essentially a partial evaluation of the %% general case below, for the single-ack case. ack([SeqId], State) -> - {#msg_status { msg_id = MsgId, - is_persistent = IsPersistent, - msg_in_store = MsgInStore, - index_on_disk = IndexOnDisk }, - State1 = #vqstate { index_state = IndexState, - msg_store_clients = MSCState, - ack_out_counter = AckOutCount }} = - remove_pending_ack(true, SeqId, State), - IndexState1 = case IndexOnDisk of - true -> rabbit_queue_index:ack([SeqId], IndexState); - false -> IndexState - end, - case MsgInStore of - true -> ok = msg_store_remove(MSCState, IsPersistent, [MsgId]); - false -> ok - end, - {[MsgId], - a(State1 #vqstate { index_state = IndexState1, - ack_out_counter = AckOutCount + 1 })}; + case remove_pending_ack(true, SeqId, State) of + {none, _} -> + State; + {#msg_status { msg_id = MsgId, + is_persistent = IsPersistent, + msg_in_store = MsgInStore, + index_on_disk = IndexOnDisk }, + State1 = #vqstate { index_state = IndexState, + msg_store_clients = MSCState, + ack_out_counter = AckOutCount }} -> + IndexState1 = case IndexOnDisk of + true -> rabbit_queue_index:ack([SeqId], IndexState); + false -> IndexState + end, + case MsgInStore of + true -> ok = msg_store_remove(MSCState, IsPersistent, [MsgId]); + false -> ok + end, + {[MsgId], + a(State1 #vqstate { index_state = IndexState1, + ack_out_counter = AckOutCount + 1 })} + end; ack(AckTags, State) -> {{IndexOnDiskSeqIds, MsgIdsByStore, AllMsgIds}, State1 = #vqstate { index_state = IndexState, @@ -659,8 +682,12 @@ ack(AckTags, State) -> ack_out_counter = AckOutCount }} = lists:foldl( fun (SeqId, {Acc, State2}) -> - {MsgStatus, State3} = remove_pending_ack(true, SeqId, State2), - {accumulate_ack(MsgStatus, Acc), State3} + case remove_pending_ack(true, SeqId, State2) of + {none, _} -> + {Acc, State2}; + {MsgStatus, State3} -> + {accumulate_ack(MsgStatus, Acc), State3} + end end, {accumulate_ack_init(), State}, AckTags), IndexState1 = rabbit_queue_index:ack(IndexOnDiskSeqIds, IndexState), remove_msgs_by_id(MsgIdsByStore, MSCState), @@ -1330,7 +1357,8 @@ init(IsDurable, IndexState, DeltaCount, DeltaBytes, Terms, io_batch_size = IoBatchSize, - mode = default }, + mode = default, + memory_reduction_run_count = 0}, a(maybe_deltas_to_betas(State)). blank_rates(Now) -> @@ -1977,8 +2005,12 @@ lookup_pending_ack(SeqId, #vqstate { ram_pending_ack = RPA, %% First parameter = UpdateStats remove_pending_ack(true, SeqId, State) -> - {MsgStatus, State1} = remove_pending_ack(false, SeqId, State), - {MsgStatus, stats({0, -1}, {MsgStatus, none}, State1)}; + case remove_pending_ack(false, SeqId, State) of + {none, _} -> + {none, State}; + {MsgStatus, State1} -> + {MsgStatus, stats({0, -1}, {MsgStatus, none}, State1)} + end; remove_pending_ack(false, SeqId, State = #vqstate{ram_pending_ack = RPA, disk_pending_ack = DPA, qi_pending_ack = QPA}) -> @@ -1990,9 +2022,13 @@ remove_pending_ack(false, SeqId, State = #vqstate{ram_pending_ack = RPA, DPA1 = gb_trees:delete(SeqId, DPA), {V, State#vqstate{disk_pending_ack = DPA1}}; none -> - QPA1 = gb_trees:delete(SeqId, QPA), - {gb_trees:get(SeqId, QPA), - State#vqstate{qi_pending_ack = QPA1}} + case gb_trees:lookup(SeqId, QPA) of + {value, V} -> + QPA1 = gb_trees:delete(SeqId, QPA), + {V, State#vqstate{qi_pending_ack = QPA1}}; + none -> + {none, State} + end end end. @@ -2143,11 +2179,15 @@ queue_merge([SeqId | Rest] = SeqIds, Q, Front, MsgIds, Limit, PubFun, State); {_, _Q1} -> %% enqueue from the remaining list of sequence ids - {MsgStatus, State1} = msg_from_pending_ack(SeqId, State), - {#msg_status { msg_id = MsgId } = MsgStatus1, State2} = - PubFun(MsgStatus, State1), - queue_merge(Rest, Q, ?QUEUE:in(MsgStatus1, Front), [MsgId | MsgIds], - Limit, PubFun, State2) + case msg_from_pending_ack(SeqId, State) of + {none, _} -> + queue_merge(Rest, Q, Front, MsgIds, Limit, PubFun, State); + {MsgStatus, State1} -> + {#msg_status { msg_id = MsgId } = MsgStatus1, State2} = + PubFun(MsgStatus, State1), + queue_merge(Rest, Q, ?QUEUE:in(MsgStatus1, Front), [MsgId | MsgIds], + Limit, PubFun, State2) + end end; queue_merge(SeqIds, Q, Front, MsgIds, _Limit, _PubFun, State) -> @@ -2156,22 +2196,28 @@ queue_merge(SeqIds, Q, Front, MsgIds, delta_merge([], Delta, MsgIds, State) -> {Delta, MsgIds, State}; delta_merge(SeqIds, Delta, MsgIds, State) -> - lists:foldl(fun (SeqId, {Delta0, MsgIds0, State0}) -> - {#msg_status { msg_id = MsgId } = MsgStatus, State1} = - msg_from_pending_ack(SeqId, State0), - {_MsgStatus, State2} = - maybe_prepare_write_to_disk(true, true, MsgStatus, State1), - {expand_delta(SeqId, Delta0), [MsgId | MsgIds0], - stats({1, -1}, {MsgStatus, none}, State2)} + lists:foldl(fun (SeqId, {Delta0, MsgIds0, State0} = Acc) -> + case msg_from_pending_ack(SeqId, State0) of + {none, _} -> + Acc; + {#msg_status { msg_id = MsgId } = MsgStatus, State1} -> + {_MsgStatus, State2} = + maybe_prepare_write_to_disk(true, true, MsgStatus, State1), + {expand_delta(SeqId, Delta0), [MsgId | MsgIds0], + stats({1, -1}, {MsgStatus, none}, State2)} + end end, {Delta, MsgIds, State}, SeqIds). %% Mostly opposite of record_pending_ack/2 msg_from_pending_ack(SeqId, State) -> - {#msg_status { msg_props = MsgProps } = MsgStatus, State1} = - remove_pending_ack(false, SeqId, State), - {MsgStatus #msg_status { - msg_props = MsgProps #message_properties { needs_confirming = false } }, - State1}. + case remove_pending_ack(false, SeqId, State) of + {none, _} -> + {none, State}; + {#msg_status { msg_props = MsgProps } = MsgStatus, State1} -> + {MsgStatus #msg_status { + msg_props = MsgProps #message_properties { needs_confirming = false } }, + State1} + end. beta_limit(Q) -> case ?QUEUE:peek(Q) of @@ -2264,6 +2310,14 @@ ifold(Fun, Acc, Its, State) -> %% Phase changes %%---------------------------------------------------------------------------- +maybe_execute_gc(State = #vqstate {memory_reduction_run_count = MRedRunCount}) -> + case MRedRunCount >= ?EXPLICIT_GC_RUN_OP_THRESHOLD of + true -> garbage_collect(), + State#vqstate{memory_reduction_run_count = 0}; + false -> State#vqstate{memory_reduction_run_count = MRedRunCount + 1} + + end. + reduce_memory_use(State = #vqstate { target_ram_count = infinity }) -> State; reduce_memory_use(State = #vqstate { @@ -2336,8 +2390,7 @@ reduce_memory_use(State = #vqstate { S2 -> push_betas_to_deltas(S2, State1) end, - garbage_collect(), - State3. + maybe_execute_gc(State3). limit_ram_acks(0, State) -> {0, ui(State)}; diff --git a/test/dynamic_ha_SUITE.erl b/test/dynamic_ha_SUITE.erl index bba7fad707..502e3a7e86 100644 --- a/test/dynamic_ha_SUITE.erl +++ b/test/dynamic_ha_SUITE.erl @@ -61,7 +61,8 @@ groups() -> ]}, {cluster_size_3, [], [ change_policy, - rapid_change + rapid_change, + nodes_policy_should_pick_master_from_its_params % FIXME: Re-enable those tests when the know issues are % fixed. %failing_random_policies, @@ -258,6 +259,48 @@ promote_on_shutdown(Config) -> durable = true}), ok. +nodes_policy_should_pick_master_from_its_params(Config) -> + [A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [all])), + %% --> Master: A + %% Slaves: [B, C] or [C, B] + Info = find_queue(?QNAME, A), + SSPids = proplists:get_value(synchronised_slave_pids, Info), + + %% Choose slave that isn't the first sync slave. Cover a bug that always + %% chose the first, even if it was not part of the policy + LastSlave = node(lists:last(SSPids)), + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [{nodes, [LastSlave]}])), + %% --> Master: B or C (depends on the order of current slaves) + %% Slaves: [] + + %% Now choose a new master that isn't synchronised. The previous + %% policy made sure that the queue only runs on one node (the last + %% from the initial synchronised list). Thus, by taking the first + %% node from this list, we know it is not synchronised. + %% + %% Because the policy doesn't cover any synchronised slave, RabbitMQ + %% should instead use an existing synchronised slave as the new master, + %% even though that isn't in the policy. + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [{nodes, [LastSlave, A]}])), + %% --> Master: B or C (same as previous policy) + %% Slaves: [A] + + NewMaster = node(erlang:hd(SSPids)), + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [{nodes, [NewMaster]}])), + %% --> Master: B or C (the other one compared to previous policy) + %% Slaves: [] + + amqp_channel:call(Ch, #'queue.delete'{queue = ?QNAME}), + _ = rabbit_ct_broker_helpers:clear_policy(Config, A, ?POLICY). + random_policy(Config) -> run_proper(fun prop_random_policy/1, [Config]). @@ -364,9 +407,8 @@ prop_random_policy(Config) -> Policies, non_empty(list(policy_gen(Nodes))), test_random_policy(Config, Nodes, Policies)). -test_random_policy(Config, Nodes, Policies) -> +apply_policy_to_declared_queue(Config, Ch, Nodes, Policies) -> [NodeA | _] = Nodes, - Ch = rabbit_ct_client_helpers:open_channel(Config, NodeA), amqp_channel:call(Ch, #'queue.declare'{queue = ?QNAME}), %% Add some load so mirrors can be busy synchronising rabbit_ct_client_helpers:publish(Ch, ?QNAME, 100000), @@ -375,7 +417,12 @@ test_random_policy(Config, Nodes, Policies) -> %% Give it some time to generate all internal notifications timer:sleep(2000), %% Check the result - Result = wait_for_last_policy(?QNAME, NodeA, Policies, 30), + wait_for_last_policy(?QNAME, NodeA, Policies, 30). + +test_random_policy(Config, Nodes, Policies) -> + [NodeA | _] = Nodes, + Ch = rabbit_ct_client_helpers:open_channel(Config, NodeA), + Result = apply_policy_to_declared_queue(Config, Ch, Nodes, Policies), %% Cleanup amqp_channel:call(Ch, #'queue.delete'{queue = ?QNAME}), _ = rabbit_ct_broker_helpers:clear_policy(Config, NodeA, ?POLICY), diff --git a/test/health_check_SUITE.erl b/test/health_check_SUITE.erl index 50abc97a02..13373d79d4 100644 --- a/test/health_check_SUITE.erl +++ b/test/health_check_SUITE.erl @@ -22,6 +22,8 @@ ,groups/0 ,init_per_suite/1 ,end_per_suite/1 + ,init_per_group/2 + ,end_per_group/2 ,init_per_testcase/2 ,end_per_testcase/2 ]). @@ -53,6 +55,10 @@ groups() -> ,ignores_stuck_remote_node_monitor ]}]. +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + init_per_suite(Config) -> rabbit_ct_helpers:log_environment(), rabbit_ct_helpers:run_setup_steps(Config). @@ -60,24 +66,30 @@ init_per_suite(Config) -> end_per_suite(Config) -> rabbit_ct_helpers:run_teardown_steps(Config). -init_per_testcase(Testcase, Config0) -> - rabbit_ct_helpers:testcase_started(Config0, Testcase), - Config1 = rabbit_ct_helpers:set_config( - Config0, [{rmq_nodes_count, 2}, - {rmq_nodes_clustered, true}]), +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = 2, + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, ClusterSize}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), rabbit_ct_helpers:run_steps(Config1, - rabbit_ct_broker_helpers:setup_steps() ++ - rabbit_ct_client_helpers:setup_steps()). + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). -end_per_testcase(Testcase, Config0) -> - Config1 = case rabbit_ct_helpers:get_config(Config0, save_config) of - undefined -> Config0; - C -> C - end, - Config2 = rabbit_ct_helpers:run_steps(Config1, - rabbit_ct_client_helpers:teardown_steps() ++ - rabbit_ct_broker_helpers:teardown_steps()), - rabbit_ct_helpers:testcase_finished(Config2, Testcase). +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). %%---------------------------------------------------------------------------- %% Test cases @@ -148,8 +160,8 @@ honors_timeout_argument(Config) -> case timer:tc(rabbit_ct_broker_helpers, rabbitmqctl, [Config, A, ["-t", "5", "node_health_check"]]) of {TimeSpent, {error, 75, _}} -> - if TimeSpent < 5000000 -> exit({too_fast, TimeSpent}); - TimeSpent > 7000000 -> exit({too_slow, TimeSpent}); %% +2 seconds for rabbitmqctl overhead + if TimeSpent < 5000000 -> exit({too_fast, TimeSpent}); + TimeSpent > 10000000 -> exit({too_slow, TimeSpent}); %% +5 seconds for rabbitmqctl overhead true -> ok end; {_, Unexpected} -> diff --git a/test/partitions_SUITE.erl b/test/partitions_SUITE.erl index aa1c1df24f..e00c015d02 100644 --- a/test/partitions_SUITE.erl +++ b/test/partitions_SUITE.erl @@ -29,6 +29,10 @@ %% passes... -define(DELAY, 8000). +%% We wait for 5 minutes for nodes to be running/blocked. +%% It's a lot, but still better than timetrap_timeout +-define(AWAIT_TIMEOUT, 300000). + all() -> [ {group, net_ticktime_1}, @@ -415,15 +419,17 @@ block(X, Y) -> allow(X, Y) -> rabbit_ct_broker_helpers:allow_traffic_between(X, Y). -await_running (Node, Bool) -> await(Node, Bool, fun is_running/1). -await_listening (Node, Bool) -> await(Node, Bool, fun is_listening/1). -await_partitions(Node, Parts) -> await(Node, Parts, fun partitions/1). +await_running (Node, Bool) -> await(Node, Bool, fun is_running/1, ?AWAIT_TIMEOUT). +await_listening (Node, Bool) -> await(Node, Bool, fun is_listening/1, ?AWAIT_TIMEOUT). +await_partitions(Node, Parts) -> await(Node, Parts, fun partitions/1, ?AWAIT_TIMEOUT). -await(Node, Res, Fun) -> +await(Node, Res, Fun, Timeout) when Timeout =< 0 -> + error({await_timeout, Node, Res, Fun}); +await(Node, Res, Fun, Timeout) -> case Fun(Node) of Res -> ok; _ -> timer:sleep(100), - await(Node, Res, Fun) + await(Node, Res, Fun, Timeout - 100) end. is_running(Node) -> rpc:call(Node, rabbit, is_running, []). diff --git a/test/unit_SUITE.erl b/test/unit_SUITE.erl index 43e812fa3d..8ac3d41afe 100644 --- a/test/unit_SUITE.erl +++ b/test/unit_SUITE.erl @@ -24,7 +24,8 @@ all() -> [ - {group, parallel_tests} + {group, parallel_tests}, + {group, sequential_tests} ]. groups() -> @@ -41,6 +42,8 @@ groups() -> ]}, content_framing, content_transcoding, + decrypt_config, + rabbitmqctl_encode, pg_local, pmerge, plmerge, @@ -59,16 +62,39 @@ groups() -> large_examples_for_size ]}, unfold, - version_equivalance, {vm_memory_monitor, [parallel], [ parse_line_linux ]} + ]}, + {sequential_tests, [], [ + decrypt_start_app, + decrypt_start_app_file, + decrypt_start_app_undefined, + decrypt_start_app_wrong_passphrase ]} ]. init_per_group(_, Config) -> Config. end_per_group(_, Config) -> Config. +init_per_testcase(TC, Config) when TC =:= decrypt_start_app; + TC =:= decrypt_start_app_file; + TC =:= decrypt_start_app_undefined -> + application:load(rabbit), + Config; +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(TC, _Config) when TC =:= decrypt_start_app; + TC =:= decrypt_start_app_file; + TC =:= decrypt_start_app_undefined -> + application:unload(rabbit), + application:unload(rabbit_shovel_test); +end_per_testcase(decrypt_config, _Config) -> + application:unload(rabbit); +end_per_testcase(_TC, _Config) -> + ok. + %% ------------------------------------------------------------------- %% Argument parsing. %% ------------------------------------------------------------------- @@ -233,6 +259,210 @@ prepend_check(HeaderKey, HeaderTable, Headers) -> rabbit_misc:table_lookup(Invalid, HeaderKey), Headers1. +decrypt_config(_Config) -> + %% Take all available block ciphers. + Hashes = rabbit_pbe:supported_hashes(), + Ciphers = rabbit_pbe:supported_ciphers(), + Iterations = [1, 10, 100, 1000], + %% Loop through all hashes, ciphers and iterations. + _ = [begin + PassPhrase = crypto:strong_rand_bytes(16), + do_decrypt_config({C, H, I, PassPhrase}) + end || H <- Hashes, C <- Ciphers, I <- Iterations], + ok. + +do_decrypt_config(Algo = {C, H, I, P}) -> + application:load(rabbit), + RabbitConfig = application:get_all_env(rabbit), + %% Encrypt a few values in configuration. + %% Common cases. + _ = [encrypt_value(Key, Algo) || Key <- [ + tcp_listeners, + num_tcp_acceptors, + ssl_options, + vm_memory_high_watermark, + default_pass, + default_permissions, + cluster_nodes, + auth_mechanisms, + msg_store_credit_disc_bound]], + %% Special case: encrypt a value in a list. + {ok, [LoopbackUser]} = application:get_env(rabbit, loopback_users), + EncLoopbackUser = rabbit_pbe:encrypt_term(C, H, I, P, LoopbackUser), + application:set_env(rabbit, loopback_users, [{encrypted, EncLoopbackUser}]), + %% Special case: encrypt a value in a key/value list. + {ok, TCPOpts} = application:get_env(rabbit, tcp_listen_options), + {_, Backlog} = lists:keyfind(backlog, 1, TCPOpts), + {_, Linger} = lists:keyfind(linger, 1, TCPOpts), + EncBacklog = rabbit_pbe:encrypt_term(C, H, I, P, Backlog), + EncLinger = rabbit_pbe:encrypt_term(C, H, I, P, Linger), + TCPOpts1 = lists:keyreplace(backlog, 1, TCPOpts, {backlog, {encrypted, EncBacklog}}), + TCPOpts2 = lists:keyreplace(linger, 1, TCPOpts1, {linger, {encrypted, EncLinger}}), + application:set_env(rabbit, tcp_listen_options, TCPOpts2), + %% Decrypt configuration. + rabbit:decrypt_config([rabbit], Algo), + %% Check that configuration was decrypted properly. + RabbitConfig = application:get_all_env(rabbit), + application:unload(rabbit), + ok. + +encrypt_value(Key, {C, H, I, P}) -> + {ok, Value} = application:get_env(rabbit, Key), + EncValue = rabbit_pbe:encrypt_term(C, H, I, P, Value), + application:set_env(rabbit, Key, {encrypted, EncValue}). + +decrypt_start_app(Config) -> + do_decrypt_start_app(Config, "hello"). + +decrypt_start_app_file(Config) -> + do_decrypt_start_app(Config, {file, ?config(data_dir, Config) ++ "/rabbit_shovel_test.passphrase"}). + +do_decrypt_start_app(Config, Passphrase) -> + %% Configure rabbit for decrypting configuration. + application:set_env(rabbit, config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000}, + {passphrase, Passphrase} + ]), + %% Add the path to our test application. + code:add_path(?config(data_dir, Config) ++ "/lib/rabbit_shovel_test/ebin"), + %% Attempt to start our test application. + %% + %% We expect a failure *after* the decrypting has been done. + try + rabbit:start_apps([rabbit_shovel_test]) + catch _:_ -> + ok + end, + %% Check if the values have been decrypted. + {ok, Shovels} = application:get_env(rabbit_shovel_test, shovels), + {_, FirstShovel} = lists:keyfind(my_first_shovel, 1, Shovels), + {_, Sources} = lists:keyfind(sources, 1, FirstShovel), + {_, Brokers} = lists:keyfind(brokers, 1, Sources), + ["amqp://fred:secret@host1.domain/my_vhost", + "amqp://john:secret@host2.domain/my_vhost"] = Brokers, + ok. + +decrypt_start_app_undefined(Config) -> + %% Configure rabbit for decrypting configuration. + application:set_env(rabbit, config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000} + %% No passphrase option! + ]), + %% Add the path to our test application. + code:add_path(?config(data_dir, Config) ++ "/lib/rabbit_shovel_test/ebin"), + %% Attempt to start our test application. + %% + %% We expect a failure during decryption because the passphrase is missing. + try + rabbit:start_apps([rabbit_shovel_test]) + catch + exit:{bad_configuration, config_entry_decoder} -> ok; + _:_ -> exit(unexpected_exception) + end. + +decrypt_start_app_wrong_passphrase(Config) -> + %% Configure rabbit for decrypting configuration. + application:set_env(rabbit, config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000}, + {passphrase, "wrong passphrase"} + ]), + %% Add the path to our test application. + code:add_path(?config(data_dir, Config) ++ "/lib/rabbit_shovel_test/ebin"), + %% Attempt to start our test application. + %% + %% We expect a failure during decryption because the passphrase is wrong. + try + rabbit:start_apps([rabbit_shovel_test]) + catch + exit:{decryption_error,_,_} -> ok; + _:_ -> exit(unexpected_exception) + end. + +rabbitmqctl_encode(_Config) -> + % list ciphers and hashes + {ok, _} = rabbit_control_pbe:encode(true, false, undefined, undefined, undefined, undefined, undefined), + {ok, _} = rabbit_control_pbe:encode(false, true, undefined, undefined, undefined, undefined, undefined), + % incorrect ciphers, hashes and iteration number + {error, _} = rabbit_control_pbe:encode(false, false, undefined, funny_cipher, undefined, undefined, undefined), + {error, _} = rabbit_control_pbe:encode(false, false, undefined, undefined, funny_hash, undefined, undefined), + {error, _} = rabbit_control_pbe:encode(false, false, undefined, undefined, undefined, -1, undefined), + {error, _} = rabbit_control_pbe:encode(false, false, undefined, undefined, undefined, 0, undefined), + % incorrect number of arguments + {error, _} = rabbit_control_pbe:encode( + false, false, + false, % encrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [] + ), + {error, _} = rabbit_control_pbe:encode( + false, false, + false, % encrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [undefined] + ), + {error, _} = rabbit_control_pbe:encode( + false, false, + false, % encrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [undefined, undefined, undefined] + ), + + % encrypt/decrypt + % string + rabbitmqctl_encode_encrypt_decrypt("foobar"), + % binary + rabbitmqctl_encode_encrypt_decrypt("<<\"foobar\">>"), + % tuple + rabbitmqctl_encode_encrypt_decrypt("{password,<<\"secret\">>}"), + + ok. + +rabbitmqctl_encode_encrypt_decrypt(Secret) -> + PassPhrase = "passphrase", + {ok, Output} = rabbit_control_pbe:encode( + false, false, + false, % encrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [Secret, PassPhrase] + ), + {encrypted, Encrypted} = rabbit_control_pbe:evaluate_input_as_term(lists:flatten(Output)), + + {ok, Result} = rabbit_control_pbe:encode( + false, false, + true, % decrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [Encrypted])), PassPhrase] + ), + Secret = lists:flatten(Result), + % decrypt with {encrypted, ...} form as input + {ok, Result} = rabbit_control_pbe:encode( + false, false, + true, % decrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [{encrypted, Encrypted}])), PassPhrase] + ), + + % wrong passphrase + {error, _} = rabbit_control_pbe:encode( + false, false, + true, % decrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [Encrypted])), PassPhrase ++ " "] + ), + {error, _} = rabbit_control_pbe:encode( + false, false, + true, % decrypt + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [{encrypted, Encrypted}])), PassPhrase ++ " "] + ) + . + %% ------------------------------------------------------------------- %% pg_local. %% ------------------------------------------------------------------- @@ -738,13 +968,3 @@ unfold(_Config) -> (N) -> {true, N*2, N-1} end, 10), passed. - -version_equivalance(_Config) -> - true = rabbit_misc:version_minor_equivalent("3.0.0", "3.0.0"), - true = rabbit_misc:version_minor_equivalent("3.0.0", "3.0.1"), - true = rabbit_misc:version_minor_equivalent("%%VSN%%", "%%VSN%%"), - false = rabbit_misc:version_minor_equivalent("3.0.0", "3.1.0"), - false = rabbit_misc:version_minor_equivalent("3.0.0", "3.0"), - false = rabbit_misc:version_minor_equivalent("3.0.0", "3.0.0.1"), - false = rabbit_misc:version_minor_equivalent("3.0.0", "3.0.foo"), - passed. diff --git a/test/unit_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app b/test/unit_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app new file mode 100644 index 0000000000..a8481c9aa4 --- /dev/null +++ b/test/unit_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app @@ -0,0 +1,46 @@ +{application, rabbit_shovel_test, + [{description, "Test .app file for tests for encrypting configuration"}, + {vsn, ""}, + {modules, []}, + {env, [ {shovels, [ {my_first_shovel, + [ {sources, + [ {brokers, [ {encrypted, <<"CfJXuka/uJYsqAtiJnwKpSY4moMPcOBh4sO8XDcdmhXbVYGKCDLKEilWPMfvOAQ2lN1BQneGn6bvDZi2+gDu6iHVKfafQAZSv8zcsVB3uYdBXFzqTCWO8TAsgG6LUMPT">>} + , {encrypted, <<"dBO6n+G1OiBwZeLXhvmNYeTE57nhBOmicUBF34zo4nQjerzQaNoEk8GA2Ts5PzMhYeO6U6Y9eEmheqIr9Gzh2duLZic65ZMQtIKNpWcZJllEhGpk7aV1COr23Yur9fWG">>} + ]} + , {declarations, [ {'exchange.declare', + [ {exchange, <<"my_fanout">>} + , {type, <<"fanout">>} + , durable + ]} + , {'queue.declare', + [{arguments, + [{<<"x-message-ttl">>, long, 60000}]}]} + , {'queue.bind', + [ {exchange, <<"my_direct">>} + , {queue, <<>>} + ]} + ]} + ]} + , {destinations, + [ {broker, "amqp://"} + , {declarations, [ {'exchange.declare', + [ {exchange, <<"my_direct">>} + , {type, <<"direct">>} + , durable + ]} + ]} + ]} + , {queue, <<>>} + , {prefetch_count, 10} + , {ack_mode, on_confirm} + , {publish_properties, [ {delivery_mode, 2} ]} + , {add_forward_headers, true} + , {publish_fields, [ {exchange, <<"my_direct">>} + , {routing_key, <<"from_shovel">>} + ]} + , {reconnect_delay, 5} + ]} + ]} + ]}, + + {applications, [kernel, stdlib]}]}. diff --git a/test/unit_SUITE_data/rabbit_shovel_test.passphrase b/test/unit_SUITE_data/rabbit_shovel_test.passphrase new file mode 100644 index 0000000000..ce01362503 --- /dev/null +++ b/test/unit_SUITE_data/rabbit_shovel_test.passphrase @@ -0,0 +1 @@ +hello |